From 82cc4afdd69efa5076579bbaf69043b99d698948 Mon Sep 17 00:00:00 2001 From: Aleksei Tikhomirov Date: Thu, 2 Mar 2023 19:25:58 +0300 Subject: [PATCH 01/73] fix_JSON_searchField --- base/base/JSON.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp index 315bcce38da..4c6d97b4444 100644 --- a/base/base/JSON.cpp +++ b/base/base/JSON.cpp @@ -466,9 +466,8 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const { if (!it->hasEscapes()) { - if (static_cast(size) + 2 > it->dataEnd() - it->data()) - continue; - if (!strncmp(data, it->data() + 1, size)) + const auto current_name = it->getRawName(); + if (current_name.size() == size && 0 == memcmp(current_name.data(), data, size)) break; } else From 808f2c0cb4bc35427c91bf9fb81f089f38e7fc84 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Wed, 8 Mar 2023 07:38:52 -0800 Subject: [PATCH 02/73] Fix hashing tuples for s390x --- src/Functions/FunctionsHashing.h | 29 +++++++++++++++++++ .../00746_hashing_tuples.reference | 14 ++++----- .../0_stateless/00746_hashing_tuples.sql | 14 ++++----- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 7b6f4213cd3..937e3b0023d 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -150,6 +150,13 @@ struct IntHash64Impl template T combineHashesFunc(T t1, T t2) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + T tmp; + reverseMemcpy(&tmp, &t1, sizeof(T)); + t1 = tmp; + reverseMemcpy(&tmp, &t2, sizeof(T)); + t2 = tmp; +#endif T hashes[] = {t1, t2}; return HashFunction::apply(reinterpret_cast(hashes), 2 * sizeof(T)); } @@ -183,6 +190,10 @@ struct HalfMD5Impl static UInt64 combineHashes(UInt64 h1, UInt64 h2) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + h1 = Poco::ByteOrder::flipBytes(static_cast(h1)); + h2 = Poco::ByteOrder::flipBytes(static_cast(h2)); +#endif UInt64 hashes[] = {h1, h2}; return apply(reinterpret_cast(hashes), 16); } @@ -322,6 +333,10 @@ struct SipHash64KeyedImpl static UInt64 combineHashesKeyed(const Key & key, UInt64 h1, UInt64 h2) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + h1 = Poco::ByteOrder::flipBytes(static_cast(h1)); + h2 = Poco::ByteOrder::flipBytes(static_cast(h2)); +#endif UInt64 hashes[] = {h1, h2}; return applyKeyed(key, reinterpret_cast(hashes), 2 * sizeof(UInt64)); } @@ -360,6 +375,13 @@ struct SipHash128KeyedImpl static UInt128 combineHashesKeyed(const Key & key, UInt128 h1, UInt128 h2) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + UInt128 tmp; + reverseMemcpy(&tmp, &h1, sizeof(UInt128)); + h1 = tmp; + reverseMemcpy(&tmp, &h2, sizeof(UInt128)); + h2 = tmp; +#endif UInt128 hashes[] = {h1, h2}; return applyKeyed(key, reinterpret_cast(hashes), 2 * sizeof(UInt128)); } @@ -395,6 +417,13 @@ struct SipHash128ReferenceKeyedImpl static UInt128 combineHashesKeyed(const Key & key, UInt128 h1, UInt128 h2) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + UInt128 tmp; + reverseMemcpy(&tmp, &h1, sizeof(UInt128)); + h1 = tmp; + reverseMemcpy(&tmp, &h2, sizeof(UInt128)); + h2 = tmp; +#endif UInt128 hashes[] = {h1, h2}; return applyKeyed(key, reinterpret_cast(hashes), 2 * sizeof(UInt128)); } diff --git a/tests/queries/0_stateless/00746_hashing_tuples.reference b/tests/queries/0_stateless/00746_hashing_tuples.reference index 71d45be5a54..e3b896f6077 100644 --- a/tests/queries/0_stateless/00746_hashing_tuples.reference +++ b/tests/queries/0_stateless/00746_hashing_tuples.reference @@ -1,11 +1,11 @@ 12940785793559895259 17926972817233444501 7456555839952096623 -CC45107CC4B79F62D831BEF2103C7CBF -DF2EC2F0669B000EDFF6ADEE264E7D68 -4CD1C30C38AB935D418B5269EF197B9E -9D78134EE48654D753CCA1B76185CF8E -389D16428D2AADEC9713905572F42864 +1 +1 +1 +1 +1 955237314186186656 8175794665478042155 9325786087413524176 @@ -18,8 +18,8 @@ DF2EC2F0669B000EDFF6ADEE264E7D68 8163029322371165472 8788309436660676487 236561483980029756 -8DD5527CC43D76F4760D26BE0F641F7E -F8F7AD9B6CD4CF117A71E277E2EC2931 +1 +1 12384823029245979431 4507350192761038840 1188926775431157506 diff --git a/tests/queries/0_stateless/00746_hashing_tuples.sql b/tests/queries/0_stateless/00746_hashing_tuples.sql index 466a2184c65..328ee5d6f05 100644 --- a/tests/queries/0_stateless/00746_hashing_tuples.sql +++ b/tests/queries/0_stateless/00746_hashing_tuples.sql @@ -4,11 +4,11 @@ SELECT sipHash64(1, 2, 3); SELECT sipHash64(1, 3, 2); SELECT sipHash64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); -SELECT hex(sipHash128('foo')); -SELECT hex(sipHash128('\x01')); -SELECT hex(sipHash128('foo', 'foo')); -SELECT hex(sipHash128('foo', 'foo', 'foo')); -SELECT hex(sipHash128(1, 2, 3)); +SELECT hex(sipHash128('foo')) = hex(reverse(unhex('CC45107CC4B79F62D831BEF2103C7CBF'))) or hex(sipHash128('foo')) = 'CC45107CC4B79F62D831BEF2103C7CBF'; +SELECT hex(sipHash128('\x01')) = hex(reverse(unhex('DF2EC2F0669B000EDFF6ADEE264E7D68'))) or hex(sipHash128('foo')) = 'DF2EC2F0669B000EDFF6ADEE264E7D68'; +SELECT hex(sipHash128('foo', 'foo')) = hex(reverse(unhex('4CD1C30C38AB935D418B5269EF197B9E'))) or hex(sipHash128('foo')) = '4CD1C30C38AB935D418B5269EF197B9E'; +SELECT hex(sipHash128('foo', 'foo', 'foo')) = hex(reverse(unhex('9D78134EE48654D753CCA1B76185CF8E'))) or hex(sipHash128('foo')) = '9D78134EE48654D753CCA1B76185CF8E'; +SELECT hex(sipHash128(1, 2, 3)) = hex(reverse(unhex('389D16428D2AADEC9713905572F42864'))) or hex(sipHash128('foo')) = '389D16428D2AADEC9713905572F42864'; SELECT halfMD5(1, 2, 3); SELECT halfMD5(1, 3, 2); @@ -26,8 +26,8 @@ SELECT murmurHash3_64(1, 2, 3); SELECT murmurHash3_64(1, 3, 2); SELECT murmurHash3_64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); -SELECT hex(murmurHash3_128('foo', 'foo')); -SELECT hex(murmurHash3_128('foo', 'foo', 'foo')); +SELECT hex(murmurHash3_128('foo', 'foo')) = hex(reverse(unhex('8DD5527CC43D76F4760D26BE0F641F7E'))) or hex(sipHash128('foo')) = '8DD5527CC43D76F4760D26BE0F641F7E'; +SELECT hex(murmurHash3_128('foo', 'foo', 'foo')) = hex(reverse(unhex('F8F7AD9B6CD4CF117A71E277E2EC2931'))) or hex(sipHash128('foo')) = 'F8F7AD9B6CD4CF117A71E277E2EC2931'; SELECT gccMurmurHash(1, 2, 3); SELECT gccMurmurHash(1, 3, 2); From 1ba75c6408830751b5d57625faa89a4164163ea4 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Thu, 16 Mar 2023 10:03:43 -0700 Subject: [PATCH 03/73] Refactored by using std::byteswap --- src/Functions/FunctionsHashing.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 937e3b0023d..a24b5109963 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -30,7 +30,7 @@ # include #endif -#include +#include #include #include #include @@ -184,15 +184,15 @@ struct HalfMD5Impl #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return buf.uint64_data; /// No need to flip bytes on big endian machines #else - return Poco::ByteOrder::flipBytes(static_cast(buf.uint64_data)); /// Compatibility with existing code. Cast need for old poco AND macos where UInt64 != uint64_t + return std::byteswap(buf.uint64_data); /// Compatibility with existing code. Cast need for old poco AND macos where UInt64 != uint64_t #endif } static UInt64 combineHashes(UInt64 h1, UInt64 h2) { #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - h1 = Poco::ByteOrder::flipBytes(static_cast(h1)); - h2 = Poco::ByteOrder::flipBytes(static_cast(h2)); + h1 = std::byteswap(h1); + h2 = std::byteswap(h2); #endif UInt64 hashes[] = {h1, h2}; return apply(reinterpret_cast(hashes), 16); @@ -334,8 +334,8 @@ struct SipHash64KeyedImpl static UInt64 combineHashesKeyed(const Key & key, UInt64 h1, UInt64 h2) { #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - h1 = Poco::ByteOrder::flipBytes(static_cast(h1)); - h2 = Poco::ByteOrder::flipBytes(static_cast(h2)); + h1 = std::byteswap(h1); + h2 = std::byteswap(h2); #endif UInt64 hashes[] = {h1, h2}; return applyKeyed(key, reinterpret_cast(hashes), 2 * sizeof(UInt64)); From f4b6205cab3d42b9058e9f594e911fb011325df2 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Thu, 16 Mar 2023 13:29:33 -0700 Subject: [PATCH 04/73] Fix sql in previous commit --- tests/queries/0_stateless/00746_hashing_tuples.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/00746_hashing_tuples.sql b/tests/queries/0_stateless/00746_hashing_tuples.sql index 328ee5d6f05..f17ad6fa77f 100644 --- a/tests/queries/0_stateless/00746_hashing_tuples.sql +++ b/tests/queries/0_stateless/00746_hashing_tuples.sql @@ -5,10 +5,10 @@ SELECT sipHash64(1, 3, 2); SELECT sipHash64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); SELECT hex(sipHash128('foo')) = hex(reverse(unhex('CC45107CC4B79F62D831BEF2103C7CBF'))) or hex(sipHash128('foo')) = 'CC45107CC4B79F62D831BEF2103C7CBF'; -SELECT hex(sipHash128('\x01')) = hex(reverse(unhex('DF2EC2F0669B000EDFF6ADEE264E7D68'))) or hex(sipHash128('foo')) = 'DF2EC2F0669B000EDFF6ADEE264E7D68'; -SELECT hex(sipHash128('foo', 'foo')) = hex(reverse(unhex('4CD1C30C38AB935D418B5269EF197B9E'))) or hex(sipHash128('foo')) = '4CD1C30C38AB935D418B5269EF197B9E'; -SELECT hex(sipHash128('foo', 'foo', 'foo')) = hex(reverse(unhex('9D78134EE48654D753CCA1B76185CF8E'))) or hex(sipHash128('foo')) = '9D78134EE48654D753CCA1B76185CF8E'; -SELECT hex(sipHash128(1, 2, 3)) = hex(reverse(unhex('389D16428D2AADEC9713905572F42864'))) or hex(sipHash128('foo')) = '389D16428D2AADEC9713905572F42864'; +SELECT hex(sipHash128('\x01')) = hex(reverse(unhex('DF2EC2F0669B000EDFF6ADEE264E7D68'))) or hex(sipHash128('\x01')) = 'DF2EC2F0669B000EDFF6ADEE264E7D68'; +SELECT hex(sipHash128('foo', 'foo')) = hex(reverse(unhex('4CD1C30C38AB935D418B5269EF197B9E'))) or hex(sipHash128('foo', 'foo')) = '4CD1C30C38AB935D418B5269EF197B9E'; +SELECT hex(sipHash128('foo', 'foo', 'foo')) = hex(reverse(unhex('9D78134EE48654D753CCA1B76185CF8E'))) or hex(sipHash128('foo', 'foo', 'foo')) = '9D78134EE48654D753CCA1B76185CF8E'; +SELECT hex(sipHash128(1, 2, 3)) = hex(reverse(unhex('389D16428D2AADEC9713905572F42864'))) or hex(sipHash128(1, 2, 3)) = '389D16428D2AADEC9713905572F42864'; SELECT halfMD5(1, 2, 3); SELECT halfMD5(1, 3, 2); @@ -26,8 +26,8 @@ SELECT murmurHash3_64(1, 2, 3); SELECT murmurHash3_64(1, 3, 2); SELECT murmurHash3_64(('a', [1, 2, 3], 4, (4, ['foo', 'bar'], 1, (1, 2)))); -SELECT hex(murmurHash3_128('foo', 'foo')) = hex(reverse(unhex('8DD5527CC43D76F4760D26BE0F641F7E'))) or hex(sipHash128('foo')) = '8DD5527CC43D76F4760D26BE0F641F7E'; -SELECT hex(murmurHash3_128('foo', 'foo', 'foo')) = hex(reverse(unhex('F8F7AD9B6CD4CF117A71E277E2EC2931'))) or hex(sipHash128('foo')) = 'F8F7AD9B6CD4CF117A71E277E2EC2931'; +SELECT hex(murmurHash3_128('foo', 'foo')) = hex(reverse(unhex('8DD5527CC43D76F4760D26BE0F641F7E'))) or hex(murmurHash3_128('foo', 'foo')) = '8DD5527CC43D76F4760D26BE0F641F7E'; +SELECT hex(murmurHash3_128('foo', 'foo', 'foo')) = hex(reverse(unhex('F8F7AD9B6CD4CF117A71E277E2EC2931'))) or hex(murmurHash3_128('foo', 'foo', 'foo')) = 'F8F7AD9B6CD4CF117A71E277E2EC2931'; SELECT gccMurmurHash(1, 2, 3); SELECT gccMurmurHash(1, 3, 2); From e805d75756fdb3650f4c2f31043286f44b3fbfef Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Mar 2023 07:10:47 +0100 Subject: [PATCH 05/73] ClickHouse Obfuscator: add README --- programs/obfuscator/Obfuscator.cpp | 1 - programs/obfuscator/README.md | 354 +++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 programs/obfuscator/README.md diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 274ad29a174..add16ec5205 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/obfuscator/README.md b/programs/obfuscator/README.md new file mode 100644 index 00000000000..8c2aaafb3ac --- /dev/null +++ b/programs/obfuscator/README.md @@ -0,0 +1,354 @@ +## clickhouse-obfuscator — a tool for dataset anonymization + +### Installation And Usage + +``` +curl https://clickhouse.com/ | sh +./clickhouse obfuscator --help +``` + +### Example + +``` +./clickhouse obfuscator --seed 123 --input-format TSV --output-format TSV \ + --structure 'CounterID UInt32, URLDomain String, URL String, SearchPhrase String, Title String' \ + < source.tsv > result.tsv +``` + + +### A long, long time ago... + +ClickHouse users already know that its biggest advantage is its high-speed processing of analytical queries. But claims like this need to be confirmed with reliable performance testing. That's what we want to talk about today. + +![benchmarks.png](https://clickhouse.com/uploads/benchmarks_24f1904cc9.png) + +We started running tests in 2013, long before ClickHouse was available as open source. Back then, our main concern was data processing speed for a web analytics product. We started storing this data, which we would later store in ClickHouse, in January 2009. Part of the data had been written to a database starting in 2012, and part was converted from OLAPServer and Metrage (data structures previously used by the solution). For testing, we took the first subset at random from data for 1 billion pageviews. Our web analytics platform didn't have any queries at that point, so we came up with queries that interested us, using all the possible ways to filter, aggregate, and sort the data. + +ClickHouse performance was compared with similar systems like Vertica and MonetDB. To avoid bias, testing was performed by an employee who hadn't participated in ClickHouse development, and special cases in the code were not optimized until all the results were obtained. We used the same approach to get a data set for functional testing. + +After ClickHouse was released as open source in 2016, people began questioning these tests. + +## Shortcomings of tests on private data + +Our performance tests: + +- Couldn't be reproduced independently because they used private data that can't be published. Some of the functional tests are not available to external users for the same reason. +- Needed further development. The set of tests needed to be substantially expanded in order to isolate performance changes in individual parts of the system. +- Didn't run on a per-commit basis or for individual pull requests. External developers couldn't check their code for performance regressions. + +We could solve these problems by throwing out the old tests and writing new ones based on open data, like [flight data for the USA](https://clickhouse.com/docs/en/getting-started/example-datasets/ontime/) and [taxi rides in New York](https://clickhouse.com/docs/en/getting-started/example-datasets/nyc-taxi). Or we could use benchmarks like TPC-H, TPC-DS, and [Star Schema Benchmark](https://clickhouse.com/docs/en/getting-started/example-datasets/star-schema). The disadvantage is that this data was very different from web analytics data, and we would rather keep the test queries. + +### Why it's important to use real data + +Performance should only be tested on real data from a production environment. Let's look at some examples. + +### Example 1 + +Let's say you fill a database with evenly distributed pseudorandom numbers. Data compression isn't going to work in this case, although data compression is essential to analytical databases. There is no silver bullet solution to the challenge of choosing the right compression algorithm and the right way to integrate it into the system since data compression requires a compromise between the speed of compression and decompression and the potential compression efficiency. But systems that can't compress data are guaranteed losers. If your tests use evenly distributed pseudorandom numbers, this factor is ignored, and the results will be distorted. + +Bottom line: Test data must have a realistic compression ratio. + +### Example 2 + +Let's say we are interested in the execution speed of this SQL query: + +```sql +SELECT RegionID, uniq(UserID) AS visitors + FROM test.hits +GROUP BY RegionID +ORDER BY visitors DESC +LIMIT 10 +``` + +This was a typical query for web analytics product. What affects the processing speed? + +- How `GROUP BY` is executed. +- Which data structure is used for calculating the `uniq` aggregate function. +- How many different RegionIDs there are and how much RAM each state of the `uniq` function requires. + +But another important factor is that the amount of data is distributed unevenly between regions. (It probably follows a power law. I put the distribution on a log-log graph, but I can't say for sure.) If this is the case, the states of the `uniq` aggregate function with fewer values must use very little memory. When there are a lot of different aggregation keys, every single byte counts. How can we get generated data that has all these properties? The obvious solution is to use real data. + +Many DBMSs implement the HyperLogLog data structure for an approximation of COUNT(DISTINCT), but none of them work very well because this data structure uses a fixed amount of memory. ClickHouse has a function that uses [a combination of three different data structures](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqcombined), depending on the size of the data set. + +Bottom line: Test data must represent distribution properties of the real data well enough, meaning cardinality (number of distinct values per column) and cross-column cardinality (number of different values counted across several different columns). + +### Example 3 + +Instead of testing the performance of the ClickHouse DBMS, let's take something simpler, like hash tables. For hash tables, it's essential to choose the right hash function. This is not as important for `std::unordered_map`, because it's a hash table based on chaining, and a prime number is used as the array size. The standard library implementation in GCC and Clang uses a trivial hash function as the default hash function for numeric types. However, `std::unordered_map` is not the best choice when we are looking for maximum speed. With an open-addressing hash table, we can't just use a standard hash function. Choosing the right hash function becomes the deciding factor. + +It's easy to find hash table performance tests using random data that don't take the hash functions used into account. Many hash function tests also focus on the calculation speed and certain quality criteria, even though they ignore the data structures used. But the fact is that hash tables and HyperLogLog require different hash function quality criteria. + +![alexey_chat.png](https://clickhouse.com/uploads/alexey_chat_3f8db88301.png) + +## Challenge + +Our goal was to obtain data for testing performance that had the same structure as our web analytics data with all the properties that are important for benchmarks, but in such a way that there remain no traces of real website users in this data. In other words, the data must be anonymized and still preserve its: + +* Compression ratio. +* Cardinality (the number of distinct values). +* Mutual cardinality between several different columns. +* Properties of probability distributions that can be used for data modeling (for example, if we believe that regions are distributed according to a power law, then the exponent — the distribution parameter — should be approximately the same for artificial data and for real data). + +How can we get a similar compression ratio for the data? If LZ4 is used, substrings in binary data must be repeated at approximately the same distance, and the repetitions must be approximately the same length. For ZSTD, entropy per byte must also coincide. + +The ultimate goal was to create a publicly available tool that anyone can use to anonymize their data sets for publication. This would allow us to debug and test performance on other people's data similar to our production data. We would also like the generated data to be interesting. + +However, these are very loosely-defined requirements, and we aren't planning to write up a formal problem statement or specification for this task. + +## Possible solutions + +I don't want to make it sound like this problem was particularly important. It was never actually included in planning, and no one had intentions to work on it. I hoped that an idea would come up someday, and suddenly I would be in a good mood and be able to put everything else off until later. + +### Explicit probabilistic models + +- We want to preserve the continuity of time series data. This means that for some types of data, we need to model the difference between neighboring values rather than the value itself. +- To model "joint cardinality" of columns, we would also have to explicitly reflect dependencies between columns. For instance, there are usually very few IP addresses per user ID, so to generate an IP address, we would have to use a hash value of the user ID as a seed and add a small amount of other pseudorandom data. +- We weren't sure how to express the dependency that the same user frequently visits URLs with matching domains at approximately the same time. + +All this can be written in a C++ "script" with the distributions and dependencies hard coded. However, Markov models are obtained from a combination of statistics with smoothing and adding noise. I started writing a script like this, but after writing explicit models for ten columns, it became unbearably boring — and the "hits" table in the web analytics product had more than 100 columns way back in 2012. + +```c++ +EventTime.day(std::discrete_distribution<>({ + 0, 0, 13, 30, 0, 14, 42, 5, 6, 31, 17, 0, 0, 0, 0, 23, 10, ...})(random)); +EventTime.hour(std::discrete_distribution<>({ + 13, 7, 4, 3, 2, 3, 4, 6, 10, 16, 20, 23, 24, 23, 18, 19, 19, ...})(random)); +EventTime.minute(std::uniform_int_distribution(0, 59)(random)); +EventTime.second(std::uniform_int_distribution(0, 59)(random)); + +UInt64 UserID = hash(4, powerLaw(5000, 1.1)); +UserID = UserID / 10000000000ULL * 10000000000ULL + + static_cast(EventTime) + UserID % 1000000; + +random_with_seed.seed(powerLaw(5000, 1.1)); +auto get_random_with_seed = [&]{ return random_with_seed(); }; +``` + +Advantages: + +- Conceptual simplicity. + +Disadvantages: + +- A large amount of work is required. +- The solution only applies to one type of data. + +And I preferred a more general solution that can be used for obfuscating any dataset. + +In any case, this solution could be improved. Instead of manually selecting models, we could implement a catalog of models and choose the best among them (best fit plus some form of regularization). Or maybe we could use Markov models for all types of fields, not just for text. Dependencies between data could also be extracted automatically. This would require calculating the [relative entropy](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) (the relative amount of information) between columns. A simpler alternative is to calculate relative cardinalities for each pair of columns (something like "how many different values of A are there on average for a fixed value B"). For instance, this will make it clear that `URLDomain` fully depends on the `URL`, and not vice versa. + +But I also rejected this idea because there are too many factors to consider, and it would take too long to write. + +### Neural networks + +As I've already mentioned, this task wasn't high on the priority list — no one was even thinking about trying to solve it. But as luck would have it, our colleague Ivan Puzirevsky was teaching at the Higher School of Economics. He asked me if I had any interesting problems that would work as suitable thesis topics for his students. When I offered him this one, he assured me it had potential. So I handed this challenge off to a nice guy "off the street" Sharif (he did have to sign an NDA to access the data, though). + +I shared all my ideas with him but emphasized that there were no restrictions on how the problem could be solved, and a good option would be to try approaches that I know nothing about, like using LSTM to generate a text dump of data. This seemed promising after coming across the article [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). + +The first challenge is that we need to generate structured data, not just text. But it wasn't clear whether a recurrent neural network could generate data with the desired structure. There are two ways to solve this. The first solution is to use separate models for generating the structure and the "filler", and only use the neural network for generating values. But this approach was postponed and then never completed. The second solution is to simply generate a TSV dump as text. Experience has shown that some of the rows in the text won't match the structure, but these rows can be thrown out when loading the data. + +The second challenge is that the recurrent neural network generates a sequence of data, and thus dependencies in data must follow in the order of the sequence. But in our data, the order of columns can potentially be in reverse to dependencies between them. We didn't do anything to resolve this problem. + +As summer approached, we had the first working Python script that generated data. The data quality seemed decent at first glance: + +![python_script.jpg](https://clickhouse.com/uploads/python_script_810d491dfb.jpg) + +However, we did run into some difficulties: + +1. The size of the model was about a gigabyte. We tried to create a model for data that was several gigabytes in size (for a start). The fact that the resulting model is so large raised concerns. Would it be possible to extract the real data that it was trained on? Unlikely. But I don't know much about machine learning and neural networks, and I haven't read this developer's Python code, so how can I be sure? There were several articles published at the time about how to compress neural networks without loss of quality, but it wasn't implemented. On the one hand, this doesn't seem to be a serious problem since we can opt out of publishing the model and just publish the generated data. On the other hand, if overfitting occurs, the generated data may contain some part of the source data. + +2. On a machine with a single CPU, the data generation speed is approximately 100 rows per second. Our goal was to generate at least a billion rows. Calculations showed that this wouldn't be completed before the date of the thesis defense. It didn't make sense to use additional hardware because the goal was to make a data generation tool that anyone could use. + +Sharif tried to analyze the quality of data by comparing statistics. Among other things, he calculated the frequency of different characters occurring in the source data and in the generated data. The result was stunning: the most frequent characters were Ð and Ñ. + +Don't worry about Sharif, though. He successfully defended his thesis, and we happily forgot about the whole thing. + +### Mutation of compressed data + +Let's assume that the problem statement has been reduced to a single point: we need to generate data that has the same compression ratio as the source data, and the data must decompress at the same speed. How can we achieve this? We need to edit compressed data bytes directly! This allows us to change the data without changing the size of the compressed data, plus everything will work fast. I wanted to try out this idea right away, despite the fact that the problem it solves is different from what we started with. But that's how it always is. + +So how do we edit a compressed file? Let's say we are only interested in LZ4. LZ4 compressed data is composed of sequences, which in turn are strings of not-compressed bytes (literals), followed by a match copy: + +1. Literals (copy the following N bytes as is). +2. Matches with a minimum repeat length of 4 (repeat N bytes in the file at a distance of M). + +Source data: + +`Hello world Hello.` + +Compressed data (arbitrary example): + +`literals 12 "Hello world " match 5 12.` + +In the compressed file, we leave "match" as-is and change the byte values in "literals". As a result, after decompressing, we get a file in which all repeating sequences at least 4 bytes long are also repeated at the same distance, but they consist of a different set of bytes (basically, the modified file doesn't contain a single byte that was taken from the source file). + +But how do we change the bytes? The answer isn't obvious because, in addition to the column types, the data also has its own internal, implicit structure that we would like to preserve. For example, text data is often stored in UTF-8 encoding, and we want the generated data also to be valid UTF-8. I developed a simple heuristic that involves meeting several criteria: + +- Null bytes and ASCII control characters are kept as-is. +- Some punctuation characters remain as-is. +- ASCII is converted to ASCII, and for everything else, the most significant bit is preserved (or an explicit set of "if" statements is written for different UTF-8 lengths). In one byte class, a new value is picked uniformly at random. +- Fragments like `https://` are preserved; otherwise, it looks a bit silly. + +The only caveat to this approach is that the data model is the source data itself, which means it cannot be published. The model is only fit for generating amounts of data no larger than the source. On the contrary, the previous approaches provide models allowing the generation of data of arbitrary size. + +``` +http://ljc.she/kdoqdqwpgafe/klwlpm&qw=962788775I0E7bs7OXeAyAx +http://ljc.she/kdoqdqwdffhant.am/wcpoyodjit/cbytjgeoocvdtclac +http://ljc.she/kdoqdqwpgafe/klwlpm&qw=962788775I0E7bs7OXe +http://ljc.she/kdoqdqwdffhant.am/wcpoyodjit/cbytjgeoocvdtclac +http://ljc.she/kdoqdqwdbknvj.s/hmqhpsavon.yf#aortxqdvjja +http://ljc.she/kdoqdqw-bknvj.s/hmqhpsavon.yf#aortxqdvjja +http://ljc.she/kdoqdqwpdtu-Unu-Rjanjna-bbcohu_qxht +http://ljc.she/kdoqdqw-bknvj.s/hmqhpsavon.yf#aortxqdvjja +http://ljc.she/kdoqdqwpdtu-Unu-Rjanjna-bbcohu_qxht +http://ljc.she/kdoqdqw-bknvj.s/hmqhpsavon.yf#aortxqdvjja +http://ljc.she/kdoqdqwpdtu-Unu-Rjanjna-bbcohu-702130 +``` + +The results were positive, and the data was interesting, but something wasn't quite right. The URLs kept the same structure, but in some of them, it was too easy to recognize the original terms, such as "avito" (a popular marketplace in Russia), so I created a heuristic that swapped some of the bytes around. + +There were other concerns as well. For example, sensitive information could possibly reside in a FixedString column in binary representation and potentially consist of ASCII control characters and punctuation, which I decided to preserve. However, I didn't take data types into consideration. + +Another problem is that if a column stores data in the "length, value" format (this is how String columns are stored), how do I ensure that the length remains correct after the mutation? When I tried to fix this, I immediately lost interest. + +### Random permutations + +Unfortunately, the problem wasn't solved. We performed a few experiments, and it just got worse. The only thing left was to sit around doing nothing and surf the web randomly since the magic was gone. Luckily, I came across a page that [explained the algorithm](http://fabiensanglard.net/fizzlefade/index.php) for rendering the death of the main character in the game Wolfenstein 3D. + +wolfenstein.gif + +
+ +The animation is really well done — the screen fills up with blood. The article explains that this is actually a pseudorandom permutation. A random permutation of a set of elements is a randomly picked bijective (one-to-one) transformation of the set. In other words, a mapping where each and every derived element corresponds to exactly one original element (and vice versa). In other words, it is a way to randomly iterate through all the elements of a data set. And that is exactly the process shown in the picture: each pixel is filled in random order, without any repetition. If we were to just choose a random pixel at each step, it would take a long time to get to the last one. + +The game uses a very simple algorithm for pseudorandom permutation called linear feedback shift register ([LFSR](https://en.wikipedia.org/wiki/Linear-feedback_shift_register)). Similar to pseudorandom number generators, random permutations, or rather their families, can be cryptographically strong when parametrized by a key. This is exactly what we needed for our data transformation. However, the details were trickier. For example, cryptographically strong encryption of N bytes to N bytes with a pre-determined key and initialization vector seems like it would work for a pseudorandom permutation of a set of N-byte strings. Indeed, this is a one-to-one transformation, and it appears to be random. But if we use the same transformation for all of our data, the result may be susceptible to cryptoanalysis because the same initialization vector and key value are used multiple times. This is similar to the [Electronic Codebook](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#ECB) mode of operation for a block cipher. + +For example, three multiplications and two xorshift operations are used for the [murmurhash](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/src/Common/HashTable/Hash.h#L18) finalizer. This operation is a pseudorandom permutation. However, I should point out that hash functions don't have to be one-to-one (even hashes of N bits to N bits). + +Or here's another interesting [example from elementary number theory](https://preshing.com/20121224/how-to-generate-a-sequence-of-unique-random-integers/) from Jeff Preshing's website. + +How can we use pseudorandom permutations to solve our problem? We can use them to transform all numeric fields so we can preserve the cardinalities and mutual cardinalities of all combinations of fields. In other words, COUNT(DISTINCT) will return the same value as before the transformation and, furthermore, with any GROUP BY. + +It is worth noting that preserving all cardinalities somewhat contradicts our goal of data anonymization. Let's say someone knows that the source data for site sessions contains a user who visited sites from 10 different countries, and they want to find that user in the transformed data. The transformed data also shows that the user visited sites from 10 different countries, which makes it easy to narrow down the search. However, even if they find out what the user was transformed into, it won't be very useful; all of the other data has also been transformed, so they won't be able to figure out what sites the user visited or anything else. But these rules can be applied in a chain. For example, suppose someone knows that the most frequently occurring website in our data is Google, with Yahoo in second place. In that case, they can use the ranking to determine which transformed site identifiers actually mean Yahoo and Google. There's nothing surprising about this since we are working with an informal problem statement, and we are trying to find a balance between the anonymization of data (hiding information) and preserving data properties (disclosure of information). For information about how to approach the data anonymization issue more reliably, read this [article](https://medium.com/georgian-impact-blog/a-brief-introduction-to-differential-privacy-eacf8722283b). + +In addition to keeping the original cardinality of values, I also wanted to keep the order of magnitude of the values. What I mean is that if the source data contained numbers under 10, then I want the transformed numbers to also be small. How can we achieve this? + +For example, we can divide a set of possible values into size classes and perform permutations within each class separately (maintaining the size classes). The easiest way to do this is to take the nearest power of two or the position of the most significant bit in the number as the size class (these are the same thing). The numbers 0 and 1 will always remain as is. The numbers 2 and 3 will sometimes remain as is (with a probability of 1/2) and will sometimes be swapped (with a probability of 1/2). The set of numbers 1024..2047 will be mapped to one of 1024! (factorial) variants, and so on. For signed numbers, we will keep the sign. + +It's also doubtful whether we need a one-to-one function. We can probably just use a cryptographically strong hash function. The transformation won't be one-to-one, but the cardinality will be close to the same. + +However, we need a cryptographically strong random permutation so that when we define a key and derive a permutation with that key, restoring the original data from the rearranged data without knowing the key would be difficult. + +There is one problem: in addition to knowing nothing about neural networks and machine learning, I am also quite ignorant when it comes to cryptography. That leaves just my courage. I was still reading random web pages and found a link on [Hackers News](https://news.ycombinator.com/item?id=15122540) to a discussion on Fabien Sanglard's page. It had a link to a [blog post](http://antirez.com/news/113) by Redis developer Salvatore Sanfilippo that talked about using a wonderful generic way of getting random permutations, known as a [Feistel network](https://en.wikipedia.org/wiki/Feistel_cipher). + +The Feistel network is iterative, consisting of rounds. Each round is a remarkable transformation that allows you to get a one-to-one function from any function. Let's look at how it works. + +1. The argument's bits are divided into two halves: +``` + arg: xxxxyyyy + arg_l: xxxx + arg_r: yyyy +``` +2. The right half replaces the left. In its place, we put the result of XOR on the initial value of the left half and the result of the function applied to the initial value of the right half, like this: + + ``` + res: yyyyzzzz + res_l = yyyy = arg_r + res_r = zzzz = arg_l ^ F(arg_r) +``` + +There is also a claim that if we use a cryptographically strong pseudorandom function for F and apply a Feistel round at least four times, we'll get a cryptographically strong pseudorandom permutation. + +This is like a miracle: we take a function that produces random garbage based on data, insert it into the Feistel network, and we now have a function that produces random garbage based on data, but yet is invertible! + +The Feistel network is at the heart of several data encryption algorithms. What we're going to do is something like encryption, only it's really bad. There are two reasons for this: + +1. We are encrypting individual values independently and in the same way, similar to the Electronic Codebook mode of operation. +2. We are storing information about the order of magnitude (the nearest power of two) and the sign of the value, which means that some values do not change at all. + +This way, we can obfuscate numeric fields while preserving the properties we need. For example, after using LZ4, the compression ratio should remain approximately the same because the duplicate values in the source data will be repeated in the converted data and at the same distances from each other. + +### Markov models + +Text models are used for data compression, predictive input, speech recognition, and random string generation. A text model is a probability distribution of all possible strings. Let's say we have an imaginary probability distribution of the texts of all the books that humanity could ever write. To generate a string, we just take a random value with this distribution and return the resulting string (a random book that humanity could write). But how do we find out the probability distribution of all possible strings? + +First, this would require too much information. There are 256^10 possible strings that are 10 bytes in length, and it would take quite a lot of memory to explicitly write a table with the probability of each string. Second, we don't have enough statistics to accurately assess the distribution. + +This is why we use a probability distribution obtained from rough statistics as the text model. For example, we could calculate the probability of each letter occurring in the text and then generate strings by selecting each next letter with the same probability. This primitive model works, but the strings are still very unnatural. + +To improve the model slightly, we could also make use of the conditional probability of the letter's occurrence if it is preceded by N-specific letters. N is a pre-set constant. Let's say N = 5, and we are calculating the probability of the letter "e" occurring after the letters "compr". This text model is called an Order-N Markov model. + +``` +P(cata | cat) = 0.8 +P(catb | cat) = 0.05 +P(catc | cat) = 0.1 +... +``` + +Let's look at how Markov models work on the website [of Hay Kranen](https://projects.haykranen.nl/markov/demo/). Unlike LSTM neural networks, the models only have enough memory for a small context of fixed-length N, so they generate funny nonsensical texts. Markov models are also used in primitive methods for generating spam, and the generated texts can be easily distinguished from real ones by counting statistics that don't fit the model. There is one advantage: Markov models work much faster than neural networks, which is exactly what we need. + +Example for Title (our examples are in Turkish because of the data used): + +
+

Hyunday Butter'dan anket shluha — Politika head manşetleri | STALKER BOXER Çiftede book — Yanudistkarışmanlı Mı Kanal | League el Digitalika Haberler Haberleri — Haberlerisi — Hotels with Centry'ler Neden babah.com

+
+ +We can calculate statistics from the source data, create a Markov model, and generate new data. Note that the model needs smoothing to avoid disclosing information about rare combinations in the source data, but this is not a problem. We use a combination of models from 0 to N. If statistics are insufficient for order N, the N−1 model is used instead. + +But we still want to preserve the cardinality of data. In other words, if the source data had 123456 unique URL values, the result should have approximately the same number of unique values. We can use a deterministically initialized random number generator to achieve this. The easiest way is to use a hash function and apply it to the original value. In other words, we get a pseudorandom result that is explicitly determined by the original value. + +Another requirement is that the source data may have many different URLs that start with the same prefix but aren't identical. For example: `https://www.clickhouse.com/images/cats/?id=xxxxxx`. We want the result to also have URLs that all start with the same prefix, but a different one. For example: http://ftp.google.kz/cgi-bin/index.phtml?item=xxxxxx. As a random number generator for generating the next character using a Markov model, we'll take a hash function from a moving window of 8 bytes at the specified position (instead of taking it from the entire string). + +
+

+https://www.clickhouse.com/images/cats/?id=12345
+                      ^^^^^^^^
+
+distribution: [aaaa][b][cc][dddd][e][ff][ggggg][h]...
+hash("images/c") % total_count:             ^
+

+
+ + It turns out to be exactly what we need. Here's the example of page titles: + +
+

+PhotoFunia - Haber7 - Hava mükemment.net Oynamak içinde şaşıracak haber, Oyunu Oynanılmaz • apród.hu kínálatában - RT Arabic
+PhotoFunia - Kinobar.Net - apród: Ingyenes | Posti
+PhotoFunia - Peg Perfeo - Castika, Sıradışı Deniz Lokoning Your Code, sire Eminema.tv/
+PhotoFunia - TUT.BY - Your Ayakkanın ve Son Dakika Spor,
+PhotoFunia - big film izle, Del Meireles offilim, Samsung DealeXtreme Değerler NEWSru.com.tv, Smotri.com Mobile yapmak Okey
+PhotoFunia 5 | Galaxy, gt, după ce anal bilgi yarak Ceza RE050A V-Stranç
+PhotoFunia :: Miami olacaksını yerel Haberler Oyun Young video
+PhotoFunia Monstelli'nin En İyi kisa.com.tr –Star Thunder Ekranı
+PhotoFunia Seks - Politika,Ekonomi,Spor GTA SANAYİ VE
+PhotoFunia Taker-Rating Star TV Resmi Söylenen Yatağa każdy dzież wierzchnie
+PhotoFunia TourIndex.Marketime oyunu Oyna Geldolları Mynet Spor,Magazin,Haberler yerel Haberleri ve Solvia, korkusuz Ev SahneTv
+PhotoFunia todo in the Gratis Perky Parti'nin yapıyı bu fotogram
+PhotoFunian Dünyasın takımız halles en kulları - TEZ
+

+
+ +## Results + +After trying four methods, I got so tired of this problem that it was time just to choose something, make it into a usable tool, and announce the solution. I chose the solution that uses random permutations and Markov models parametrized by a key. It is implemented as the clickhouse-obfuscator program, which is very easy to use. The input is a table dump in any supported format (such as CSV or JSONEachRow), and the command line parameters specify the table structure (column names and types) and the secret key (any string, which you can forget immediately after use). The output is the same number of rows of obfuscated data. + +The program is installed with `clickhouse-client`, has no dependencies, and works on almost any flavor of Linux. You can apply it to any database dump, not just ClickHouse. For instance, you can generate test data from MySQL or PostgreSQL databases or create development databases that are similar to your production databases. + +```bash +clickhouse-obfuscator \ + --seed "$(head -c16 /dev/urandom | base64)" \ + --input-format TSV --output-format TSV \ + --structure 'CounterID UInt32, URLDomain String, \ + URL String, SearchPhrase String, Title String' \ + < table.tsv > result.tsv +``` + +```bash + clickhouse-obfuscator --help +``` + +Of course, everything isn't so cut and dry because data transformed by this program is almost completely reversible. The question is whether it is possible to perform the reverse transformation without knowing the key. If the transformation used a cryptographic algorithm, this operation would be as difficult as a brute-force search. Although the transformation uses some cryptographic primitives, they are not used in the correct way, and the data is susceptible to certain methods of analysis. To avoid problems, these issues are covered in the documentation for the program (access it using --help). + +In the end, we transformed the data set we needed [for functional and performance testing](https://clickhouse.com/docs/en/getting-started/example-datasets/metrica/), and received approval from our data security team to publish. + +Our developers and members of our community use this data for real performance testing when optimizing algorithms inside ClickHouse. Third-party users can provide us with their obfuscated data so that we can make ClickHouse even faster for them. We also released an independent open benchmark for hardware and cloud providers on top of this data: [https://benchmark.clickhouse.com/](https://benchmark.clickhouse.com/) From 10980554091440304b8a235079947e685f2c3ad5 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:21:10 +0000 Subject: [PATCH 06/73] Stop `wait for quorum` retries on shutdown --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 035cbdac55e..43e7e1e3979 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -1073,13 +1073,26 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( } }, [&zookeeper]() { zookeeper->cleanupEphemeralNodes(); }); + if (!conflict_block_ids.empty()) return conflict_block_ids; + if (isQuorumEnabled()) { ZooKeeperRetriesControl quorum_retries_ctl("waitForQuorum", zookeeper_retries_info); quorum_retries_ctl.retryLoop([&]() { + if (storage.is_readonly) + { + /// stop retries if in shutdown + if (storage.shutdown_called) + throw Exception( + ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + + quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); + return; + } + zookeeper->setKeeper(storage.getZooKeeper()); if (is_already_existing_part) From 795a1c84e1d77390dede017756af701250058039 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:48:04 +0000 Subject: [PATCH 07/73] Actually stop retries in case of storage shutdown --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 43e7e1e3979..600e2f408ef 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -649,8 +649,11 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) + { + retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + } /// When we attach existing parts it's okay to be in read-only mode /// For example during RESTORE REPLICA. @@ -1086,8 +1089,11 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) + { + quorum_retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + } quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); return; From 1073c7f3c942949d1f5de6956fab9b3bcde8b34b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:54:25 +0000 Subject: [PATCH 08/73] Revert "Actually stop retries in case of storage shutdown" This was unnecessary since, as for exceptions, retries done only on zkutil::KeeperException This reverts commit 795a1c84e1d77390dede017756af701250058039. --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 600e2f408ef..43e7e1e3979 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -649,11 +649,8 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) - { - retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); - } /// When we attach existing parts it's okay to be in read-only mode /// For example during RESTORE REPLICA. @@ -1089,11 +1086,8 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) - { - quorum_retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); - } quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); return; From 038bfb40ab7fd66c693317f957dd4be1a43eb280 Mon Sep 17 00:00:00 2001 From: AVMusorin Date: Mon, 27 Feb 2023 15:42:38 +0100 Subject: [PATCH 09/73] dynamic update system.backups --- src/Backups/BackupUtils.cpp | 137 --------------------------- src/Backups/BackupUtils.h | 9 -- src/Backups/BackupsWorker.cpp | 171 +++++++++++++++++++++++++++++++--- src/Backups/BackupsWorker.h | 12 +++ 4 files changed, 172 insertions(+), 157 deletions(-) diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index c6a0840964b..89b75a103c2 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -1,10 +1,7 @@ #include -#include -#include #include #include #include -#include #include @@ -60,140 +57,6 @@ DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & e } -void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool) -{ - size_t num_active_jobs = 0; - std::mutex mutex; - std::condition_variable event; - std::exception_ptr exception; - - bool always_single_threaded = !backup->supportsWritingInMultipleThreads(); - auto thread_group = CurrentThread::getGroup(); - - for (auto & name_and_entry : backup_entries) - { - auto & name = name_and_entry.first; - auto & entry = name_and_entry.second; - - { - std::unique_lock lock{mutex}; - if (exception) - break; - ++num_active_jobs; - } - - auto job = [&](bool async) - { - SCOPE_EXIT_SAFE( - std::lock_guard lock{mutex}; - if (!--num_active_jobs) - event.notify_all(); - if (async) - CurrentThread::detachFromGroupIfNotDetached(); - ); - - try - { - if (async && thread_group) - CurrentThread::attachToGroup(thread_group); - - if (async) - setThreadName("BackupWorker"); - - { - std::lock_guard lock{mutex}; - if (exception) - return; - } - - backup->writeFile(name, std::move(entry)); - } - catch (...) - { - std::lock_guard lock{mutex}; - if (!exception) - exception = std::current_exception(); - } - }; - - if (always_single_threaded || !thread_pool.trySchedule([job] { job(true); })) - job(false); - } - - { - std::unique_lock lock{mutex}; - event.wait(lock, [&] { return !num_active_jobs; }); - if (exception) - std::rethrow_exception(exception); - } -} - - -void restoreTablesData(DataRestoreTasks && tasks, ThreadPool & thread_pool) -{ - size_t num_active_jobs = 0; - std::mutex mutex; - std::condition_variable event; - std::exception_ptr exception; - - auto thread_group = CurrentThread::getGroup(); - - for (auto & task : tasks) - { - { - std::unique_lock lock{mutex}; - if (exception) - break; - ++num_active_jobs; - } - - auto job = [&](bool async) - { - SCOPE_EXIT_SAFE( - std::lock_guard lock{mutex}; - if (!--num_active_jobs) - event.notify_all(); - if (async) - CurrentThread::detachFromGroupIfNotDetached(); - ); - - try - { - if (async && thread_group) - CurrentThread::attachToGroup(thread_group); - - if (async) - setThreadName("RestoreWorker"); - - { - std::lock_guard lock{mutex}; - if (exception) - return; - } - - std::move(task)(); - } - catch (...) - { - std::lock_guard lock{mutex}; - if (!exception) - exception = std::current_exception(); - } - }; - - if (!thread_pool.trySchedule([job] { job(true); })) - job(false); - } - - { - std::unique_lock lock{mutex}; - event.wait(lock, [&] { return !num_active_jobs; }); - if (exception) - std::rethrow_exception(exception); - } -} - - /// Returns access required to execute BACKUP query. AccessRightsElements getRequiredAccessToBackup(const ASTBackupQuery::Elements & elements) { diff --git a/src/Backups/BackupUtils.h b/src/Backups/BackupUtils.h index cda9121b1fa..f451b003652 100644 --- a/src/Backups/BackupUtils.h +++ b/src/Backups/BackupUtils.h @@ -7,21 +7,12 @@ namespace DB { class IBackup; -using BackupMutablePtr = std::shared_ptr; -class IBackupEntry; -using BackupEntries = std::vector>>; -using DataRestoreTasks = std::vector>; class AccessRightsElements; class DDLRenamingMap; /// Initializes a DDLRenamingMap from a BACKUP or RESTORE query. DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & elements); -/// Write backup entries to an opened backup. -void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool); - -/// Run data restoring tasks which insert data to tables. -void restoreTablesData(DataRestoreTasks && tasks, ThreadPool & thread_pool); /// Returns access required to execute BACKUP query. AccessRightsElements getRequiredAccessToBackup(const ASTBackupQuery::Elements & elements); diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index bdcff249e7d..58185053124 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace DB @@ -346,7 +347,7 @@ void BackupsWorker::doBackup( } /// Write the backup entries to the backup. - writeBackupEntries(backup, std::move(backup_entries), backups_thread_pool); + writeBackupEntries(backup_id, backup, std::move(backup_entries), backups_thread_pool, backup_settings.internal); /// We have written our backup entries, we need to tell other hosts (they could be waiting for it). backup_coordination->setStage(backup_settings.host_id, Stage::COMPLETED, ""); @@ -374,6 +375,7 @@ void BackupsWorker::doBackup( LOG_INFO(log, "{} {} was created successfully", (backup_settings.internal ? "Internal backup" : "Backup"), backup_name_for_logging); setStatus(backup_id, BackupStatus::BACKUP_CREATED); + /// NOTE: we need to update metadata again after backup->finalizeWriting(), because backup metadata is written there. setNumFilesAndSize(backup_id, num_files, total_size, num_entries, uncompressed_size, compressed_size, 0, 0); } catch (...) @@ -394,6 +396,88 @@ void BackupsWorker::doBackup( } +void BackupsWorker::writeBackupEntries(const OperationID & backup_id, BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool, bool internal) +{ + size_t num_active_jobs = 0; + std::mutex mutex; + std::condition_variable event; + std::exception_ptr exception; + + bool always_single_threaded = !backup->supportsWritingInMultipleThreads(); + auto thread_group = CurrentThread::getGroup(); + + for (auto & name_and_entry : backup_entries) + { + auto & name = name_and_entry.first; + auto & entry = name_and_entry.second; + + { + std::unique_lock lock{mutex}; + if (exception) + break; + ++num_active_jobs; + } + + auto job = [&](bool async) + { + SCOPE_EXIT_SAFE( + std::lock_guard lock{mutex}; + if (!--num_active_jobs) + event.notify_all(); + if (async) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + try + { + if (async && thread_group) + CurrentThread::attachToGroup(thread_group); + + if (async) + setThreadName("BackupWorker"); + + { + std::lock_guard lock{mutex}; + if (exception) + return; + } + + backup->writeFile(name, std::move(entry)); + // Update metadata + if (!internal) + { + setNumFilesAndSize( + backup_id, + backup->getNumFiles(), + backup->getTotalSize(), + backup->getNumEntries(), + backup->getUncompressedSize(), + backup->getCompressedSize(), + 0, 0); + } + + } + catch (...) + { + std::lock_guard lock{mutex}; + if (!exception) + exception = std::current_exception(); + } + }; + + if (always_single_threaded || !thread_pool.trySchedule([job] { job(true); })) + job(false); + } + + { + std::unique_lock lock{mutex}; + event.wait(lock, [&] { return !num_active_jobs; }); + if (exception) + std::rethrow_exception(exception); + } +} + + OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr context) { auto restore_query = std::static_pointer_cast(query->clone()); @@ -578,7 +662,7 @@ void BackupsWorker::doRestore( } /// Execute the data restoring tasks. - restoreTablesData(std::move(data_restore_tasks), restores_thread_pool); + restoreTablesData(restore_id, backup, std::move(data_restore_tasks), restores_thread_pool); /// We have restored everything, we need to tell other hosts (they could be waiting for it). restore_coordination->setStage(restore_settings.host_id, Stage::COMPLETED, ""); @@ -586,15 +670,6 @@ void BackupsWorker::doRestore( LOG_INFO(log, "Restored from {} {} successfully", (restore_settings.internal ? "internal backup" : "backup"), backup_name_for_logging); setStatus(restore_id, BackupStatus::RESTORED); - setNumFilesAndSize( - restore_id, - backup->getNumFiles(), - backup->getTotalSize(), - backup->getNumEntries(), - backup->getUncompressedSize(), - backup->getCompressedSize(), - backup->getNumReadFiles(), - backup->getNumReadBytes()); } catch (...) { @@ -614,6 +689,80 @@ void BackupsWorker::doRestore( } +void BackupsWorker::restoreTablesData(const OperationID & restore_id, BackupPtr backup, DataRestoreTasks && tasks, ThreadPool & thread_pool) +{ + size_t num_active_jobs = 0; + std::mutex mutex; + std::condition_variable event; + std::exception_ptr exception; + + auto thread_group = CurrentThread::getGroup(); + + for (auto & task : tasks) + { + { + std::unique_lock lock{mutex}; + if (exception) + break; + ++num_active_jobs; + } + + auto job = [&](bool async) + { + SCOPE_EXIT_SAFE( + std::lock_guard lock{mutex}; + if (!--num_active_jobs) + event.notify_all(); + if (async) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + try + { + if (async && thread_group) + CurrentThread::attachToGroup(thread_group); + + if (async) + setThreadName("RestoreWorker"); + + { + std::lock_guard lock{mutex}; + if (exception) + return; + } + + std::move(task)(); + setNumFilesAndSize( + restore_id, + backup->getNumFiles(), + backup->getTotalSize(), + backup->getNumEntries(), + backup->getUncompressedSize(), + backup->getCompressedSize(), + backup->getNumReadFiles(), + backup->getNumReadBytes()); + } + catch (...) + { + std::lock_guard lock{mutex}; + if (!exception) + exception = std::current_exception(); + } + }; + + if (!thread_pool.trySchedule([job] { job(true); })) + job(false); + } + + { + std::unique_lock lock{mutex}; + event.wait(lock, [&] { return !num_active_jobs; }); + if (exception) + std::rethrow_exception(exception); + } +} + + void BackupsWorker::addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status) { Info info; diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 0f5c16cd71f..c36b58da14f 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -17,6 +17,12 @@ struct RestoreSettings; struct BackupInfo; class IBackupCoordination; class IRestoreCoordination; +class IBackup; +using BackupMutablePtr = std::shared_ptr; +using BackupPtr = std::shared_ptr; +class IBackupEntry; +using BackupEntries = std::vector>>; +using DataRestoreTasks = std::vector>; /// Manager of backups and restores: executes backups and restores' threads in the background. /// Keeps information about backups and restores started in this session. @@ -99,6 +105,9 @@ private: ContextMutablePtr mutable_context, bool called_async); + /// Write backup entries to an opened backup. + void writeBackupEntries(const OperationID & backup_id, BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool, bool internal); + OperationID startRestoring(const ASTPtr & query, ContextMutablePtr context); void doRestore( @@ -111,6 +120,9 @@ private: ContextMutablePtr context, bool called_async); + /// Run data restoring tasks which insert data to tables. + void restoreTablesData(const OperationID & restore_id, BackupPtr backup, DataRestoreTasks && tasks, ThreadPool & thread_pool); + void addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status); void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true); void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } From 62c8eeb25cf5bd55b60c3cfbaf1ce1a92fb893cd Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 22 Mar 2023 20:13:30 +0000 Subject: [PATCH 10/73] Ignoring inaccessible tables for attaching to the materilized view --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 75 +++++++++++++------ ...re_inacc_tables_mat_view_atttach.reference | 0 ...6_ignore_inacc_tables_mat_view_atttach.sql | 22 ++++++ 4 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference create mode 100644 tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index accfe387137..644234248f1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -467,6 +467,7 @@ class IColumn; M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ + M(Bool, ignore_inaccessible_tables_mat_view_attach, false, "Ignoring inaccessible tables for attaching to the materilized view.", 0) \ \ M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7a4d65a4d57..c15607bc273 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -97,6 +97,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int UNKNOWN_DATABASE; + extern const int UNKNOWN_TABLE; extern const int PATH_ACCESS_DENIED; extern const int NOT_IMPLEMENTED; extern const int ENGINE_REQUIRED; @@ -1210,35 +1211,61 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { Block input_block; - if (getContext()->getSettingsRef().allow_experimental_analyzer) + auto check_type_compatible_for_materialize_view = [&]() { - input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); + if (getContext()->getSettingsRef().allow_experimental_analyzer) + { + input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); + } + else + { + input_block = InterpreterSelectWithUnionQuery(create.select->clone(), + getContext(), + SelectQueryOptions().analyze()).getSampleBlock(); + } + + Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); + + ColumnsWithTypeAndName input_columns; + ColumnsWithTypeAndName output_columns; + for (const auto & input_column : input_block) + { + if (const auto * output_column = output_block.findByName(input_column.name)) + { + input_columns.push_back(input_column.cloneEmpty()); + output_columns.push_back(output_column->cloneEmpty()); + } + } + + ActionsDAG::makeConvertingActions( + input_columns, + output_columns, + ActionsDAG::MatchColumnsMode::Position + ); + }; + + if (getContext()->getSettingsRef().ignore_inaccessible_tables_mat_view_attach) + { + try + { + check_type_compatible_for_materialize_view(); + } catch (const Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_TABLE && create.attach) + { + auto * log = &Poco::Logger::get("InterpreterSelectQuery"); + LOG_WARNING(log, "{}", e.message()); + } + else + { + throw; + } + } } else { - input_block = InterpreterSelectWithUnionQuery(create.select->clone(), - getContext(), - SelectQueryOptions().analyze()).getSampleBlock(); + check_type_compatible_for_materialize_view(); } - - Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); - - ColumnsWithTypeAndName input_columns; - ColumnsWithTypeAndName output_columns; - for (const auto & input_column : input_block) - { - if (const auto * output_column = output_block.findByName(input_column.name)) - { - input_columns.push_back(input_column.cloneEmpty()); - output_columns.push_back(output_column->cloneEmpty()); - } - } - - ActionsDAG::makeConvertingActions( - input_columns, - output_columns, - ActionsDAG::MatchColumnsMode::Position - ); } } diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql new file mode 100644 index 00000000000..2c81eb245ee --- /dev/null +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -0,0 +1,22 @@ +DROP DATABASE IF EXISTS test_db; + +SET ignore_inaccessible_tables_mat_view_attach = 1; +SET send_logs_level = 'fatal'; + +CREATE DATABASE test_db; + +CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; + +DROP TABLE test_db.table; + +DETACH TABLE test_db.mview; + +ATTACH TABLE test_db.mview; + +SET ignore_inaccessible_tables_mat_view_attach = 0; + +DROP DATABASE test_db; \ No newline at end of file From f047e1b6682b9f01fa7bac038b8092e9d4369b0b Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Wed, 22 Mar 2023 20:18:40 +0000 Subject: [PATCH 11/73] Fixed new line --- .../0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index 2c81eb245ee..1b11ec9a9c8 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -19,4 +19,5 @@ ATTACH TABLE test_db.mview; SET ignore_inaccessible_tables_mat_view_attach = 0; -DROP DATABASE test_db; \ No newline at end of file +DROP DATABASE test_db; + From 679fe4c9cd60d4b302c8f37206b41fe4a674fa74 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 23 Mar 2023 14:57:52 +0100 Subject: [PATCH 12/73] Added test. Closes #12264 --- tests/queries/0_stateless/25402_fail_on_invalid_having.reference | 0 tests/queries/0_stateless/25402_fail_on_invalid_having.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/25402_fail_on_invalid_having.reference create mode 100644 tests/queries/0_stateless/25402_fail_on_invalid_having.sql diff --git a/tests/queries/0_stateless/25402_fail_on_invalid_having.reference b/tests/queries/0_stateless/25402_fail_on_invalid_having.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/25402_fail_on_invalid_having.sql b/tests/queries/0_stateless/25402_fail_on_invalid_having.sql new file mode 100644 index 00000000000..beb72218685 --- /dev/null +++ b/tests/queries/0_stateless/25402_fail_on_invalid_having.sql @@ -0,0 +1 @@ +SELECT a, sum(b) FROM (SELECT 1 AS a, 1 AS b, 0 AS c) GROUP BY a HAVING c -- {{ serverError NOT_AN_AGGREGATE}} \ No newline at end of file From 5e95a37c526875ad1791240895f0f5cb72632e41 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 23 Mar 2023 07:05:51 -0700 Subject: [PATCH 13/73] Fix decimal-256 text output issue on s390x --- base/base/wide_integer_impl.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 4a54c0fb2a4..874a66440a7 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -732,9 +732,10 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(rhs); + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, big(i)); + base_type rhs_item = get_item(t, big(i)); if (lhs.items[big(i)] != rhs_item) return lhs.items[big(i)] > rhs_item; @@ -757,9 +758,10 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(lhs); + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, big(i)); + base_type rhs_item = get_item(t, big(i)); if (lhs.items[big(i)] != rhs_item) return lhs.items[big(i)] < rhs_item; @@ -779,9 +781,10 @@ public: { if constexpr (should_keep_size()) { + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, any(i)); + base_type rhs_item = get_item(t, any(i)); if (lhs.items[any(i)] != rhs_item) return false; From 08583c840535529d297f7eab6d72271c4b7752b6 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 23 Mar 2023 07:17:37 -0700 Subject: [PATCH 14/73] Fixed the style issue --- base/base/wide_integer_impl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 874a66440a7..30d08be2e4f 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -732,7 +732,7 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(rhs); - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, big(i)); @@ -758,7 +758,7 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(lhs); - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, big(i)); @@ -781,7 +781,7 @@ public: { if constexpr (should_keep_size()) { - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, any(i)); From 9003b7d7893d9d8596dd6215d7cb78a9c2d92599 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:14:32 +0100 Subject: [PATCH 15/73] Fix tsan error lock-order-inversion --- src/Interpreters/ProcessList.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 5db39ece2e5..c7916f3eed2 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -434,11 +434,16 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { - std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); - assert(it != executors.end()); + ExecutorHolderPtr executor_holder; + { + std::lock_guard lock(executors_mutex); + auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + assert(it != executors.end()); + executor_holder = *it; + } /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) - (*it)->remove(); + /// We should do it with released executors_mutex to avoid possible lock order inversion. + executor_holder->remove(); } bool QueryStatus::checkTimeLimit() From 2491128b087baaf3bb5ef7c6605555ce1d2c2130 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 23 Mar 2023 17:40:04 -0300 Subject: [PATCH 16/73] Add unit test to assert arrow lib does not abort on fatal logs --- src/CMakeLists.txt | 1 + .../gtest_assert_arrow_log_does_not_abort.cpp | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index edf638c5350..e5cdd14d826 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -608,6 +608,7 @@ if (ENABLE_TESTS) dbms clickhouse_common_config clickhouse_common_zookeeper + ch_contrib::parquet string_utils) if (TARGET ch_contrib::simdjson) diff --git a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp new file mode 100644 index 00000000000..93bcdb29f73 --- /dev/null +++ b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp @@ -0,0 +1,15 @@ +#include +#include +#include +#include + +using namespace DB; + +TEST(ChunkedArray, ChunkedArrayWithZeroChunksShouldNotAbort) +{ + std::vector> empty_chunks_vector; + + EXPECT_ANY_THROW(::arrow::ChunkedArray{empty_chunks_vector}); + + ::arrow::util::ArrowLog(__FILE__, __LINE__, ::arrow::util::ArrowLogLevel::ARROW_FATAL); +} From d13aa3c02962c90790a3705b6b4898f4370e87f5 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 23 Mar 2023 17:42:55 -0300 Subject: [PATCH 17/73] missing change --- .../tests/gtest_assert_arrow_log_does_not_abort.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp index 93bcdb29f73..fb13f4664d9 100644 --- a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp +++ b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp @@ -10,6 +10,9 @@ TEST(ChunkedArray, ChunkedArrayWithZeroChunksShouldNotAbort) std::vector> empty_chunks_vector; EXPECT_ANY_THROW(::arrow::ChunkedArray{empty_chunks_vector}); - - ::arrow::util::ArrowLog(__FILE__, __LINE__, ::arrow::util::ArrowLogLevel::ARROW_FATAL); +} + +TEST(ArrowLog, FatalLogShouldThrow) +{ + EXPECT_ANY_THROW(::arrow::util::ArrowLog(__FILE__, __LINE__, ::arrow::util::ArrowLogLevel::ARROW_FATAL)); } From 68118388e6a3f73a1091a29a353d5525953f7cb1 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Fri, 24 Mar 2023 08:23:22 +0000 Subject: [PATCH 18/73] Rename opt key, added test and refactored code. --- src/Interpreters/InterpreterCreateQuery.cpp | 126 +++++++++--------- src/Interpreters/InterpreterCreateQuery.h | 3 + ...6_ignore_inacc_tables_mat_view_atttach.sql | 7 +- ...re_inacc_tables_mat_view_atttach.reference | 0 ...t_ignore_inacc_tables_mat_view_atttach.sql | 20 +++ ..._view_state_after_droping_tables.reference | 2 + ...re_mat_view_state_after_droping_tables.sql | 27 ++++ 7 files changed, 118 insertions(+), 67 deletions(-) create mode 100644 tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference create mode 100644 tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql create mode 100644 tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference create mode 100644 tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index c15607bc273..67dd053ef19 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1077,6 +1077,68 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data } } +Block InterpreterCreateQuery::getSampleBlockFromCreteQuery(const ASTCreateQuery & create) +{ + Block result_block; + if (getContext()->getSettingsRef().allow_experimental_analyzer) + { + result_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); + } + else + { + result_block = InterpreterSelectWithUnionQuery(create.select->clone(), + getContext(), + SelectQueryOptions().analyze()).getSampleBlock(); + } + return result_block; +} + +void InterpreterCreateQuery::checkTypecompatibleForMaterializeView(const ASTCreateQuery & create) +{ + if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( + {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid}, + getContext() + )) + { + Block input_block; + try + { + input_block = getSampleBlockFromCreteQuery(create); + } + catch (const Exception & e) + { + if (getContext()->getSettingsRef().skip_materialized_view_checking_if_source_table_not_exist && + e.code() == ErrorCodes::UNKNOWN_TABLE && create.attach + ) + { + LOG_WARNING(&Poco::Logger::get("InterpreterSelectQuery"), "{}", e.message()); + } + else + { + throw; + } + } + + Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); + + ColumnsWithTypeAndName input_columns; + ColumnsWithTypeAndName output_columns; + for (const auto & input_column : input_block) + { + if (const auto * output_column = output_block.findByName(input_column.name)) + { + input_columns.push_back(input_column.cloneEmpty()); + output_columns.push_back(output_column->cloneEmpty()); + } + } + + ActionsDAG::makeConvertingActions( + input_columns, + output_columns, + ActionsDAG::MatchColumnsMode::Position + ); + } +} BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { @@ -1204,69 +1266,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// Check type compatible for materialized dest table and select columns if (create.select && create.is_materialized_view && create.to_table_id) { - if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( - {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid}, - getContext() - )) - { - Block input_block; - - auto check_type_compatible_for_materialize_view = [&]() - { - if (getContext()->getSettingsRef().allow_experimental_analyzer) - { - input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); - } - else - { - input_block = InterpreterSelectWithUnionQuery(create.select->clone(), - getContext(), - SelectQueryOptions().analyze()).getSampleBlock(); - } - - Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); - - ColumnsWithTypeAndName input_columns; - ColumnsWithTypeAndName output_columns; - for (const auto & input_column : input_block) - { - if (const auto * output_column = output_block.findByName(input_column.name)) - { - input_columns.push_back(input_column.cloneEmpty()); - output_columns.push_back(output_column->cloneEmpty()); - } - } - - ActionsDAG::makeConvertingActions( - input_columns, - output_columns, - ActionsDAG::MatchColumnsMode::Position - ); - }; - - if (getContext()->getSettingsRef().ignore_inaccessible_tables_mat_view_attach) - { - try - { - check_type_compatible_for_materialize_view(); - } catch (const Exception & e) - { - if (e.code() == ErrorCodes::UNKNOWN_TABLE && create.attach) - { - auto * log = &Poco::Logger::get("InterpreterSelectQuery"); - LOG_WARNING(log, "{}", e.message()); - } - else - { - throw; - } - } - } - else - { - check_type_compatible_for_materialize_view(); - } - } + checkTypecompatibleForMaterializeView(create); } DatabasePtr database; diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 4d11387f44c..0d3ce92a4ff 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -95,6 +95,9 @@ private: BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties); /// Inserts data in created table if it's CREATE ... SELECT BlockIO fillTableIfNeeded(const ASTCreateQuery & create); + /// Check type compatible for materialized dest table and select columns + void checkTypecompatibleForMaterializeView(const ASTCreateQuery & create); + Block getSampleBlockFromCreteQuery(const ASTCreateQuery & create); void assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const; diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index 1b11ec9a9c8..854a074c234 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS test_db; -SET ignore_inaccessible_tables_mat_view_attach = 1; +SET skip_materialized_view_checking_if_source_table_not_exist = 1; SET send_logs_level = 'fatal'; CREATE DATABASE test_db; @@ -17,7 +17,6 @@ DETACH TABLE test_db.mview; ATTACH TABLE test_db.mview; -SET ignore_inaccessible_tables_mat_view_attach = 0; - -DROP DATABASE test_db; +SET skip_materialized_view_checking_if_source_table_not_exist = 0; +DROP DATABASE test_db; \ No newline at end of file diff --git a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql new file mode 100644 index 00000000000..146952b263b --- /dev/null +++ b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql @@ -0,0 +1,20 @@ +DROP DATABASE IF EXISTS test_db; + +SET skip_materialized_view_checking_if_source_table_not_exist = 0; +SET send_logs_level = 'fatal'; + +CREATE DATABASE test_db; + +CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; + +DROP TABLE test_db.table; + +DETACH TABLE test_db.mview; + +ATTACH TABLE test_db.mview; --{serverError 60} + +DROP DATABASE test_db; \ No newline at end of file diff --git a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference new file mode 100644 index 00000000000..bb3ee860aec --- /dev/null +++ b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference @@ -0,0 +1,2 @@ +3 some_val +3 9 diff --git a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql new file mode 100644 index 00000000000..43c4efb8843 --- /dev/null +++ b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql @@ -0,0 +1,27 @@ +DROP DATABASE IF EXISTS test_db; + +SET skip_materialized_view_checking_if_source_table_not_exist = 1; +SET send_logs_level = 'fatal'; + +CREATE DATABASE test_db; + +CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; + +CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; + +DROP TABLE test_db.table; + +DETACH TABLE test_db.mview; + +ATTACH TABLE test_db.mview; + +CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; + +INSERT INTO test_db.table VALUES (3,'some_val'); + +SELECT n,s FROM test_db.table ORDER BY n; +SELECT n,n2 FROM test_db.mview ORDER by n; + +DROP DATABASE test_db; \ No newline at end of file From abd9a5bef60c0767fd3c1801cb5ee064fa7c9160 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Fri, 24 Mar 2023 08:25:48 +0000 Subject: [PATCH 19/73] Fixed new lines in test --- .../0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql | 3 ++- .../02697_do_not_ignore_inacc_tables_mat_view_atttach.sql | 3 ++- .../02698_restore_mat_view_state_after_droping_tables.sql | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index 854a074c234..cfb363053e5 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -19,4 +19,5 @@ ATTACH TABLE test_db.mview; SET skip_materialized_view_checking_if_source_table_not_exist = 0; -DROP DATABASE test_db; \ No newline at end of file +DROP DATABASE test_db; + diff --git a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql index 146952b263b..492c2ed1fa2 100644 --- a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql @@ -17,4 +17,5 @@ DETACH TABLE test_db.mview; ATTACH TABLE test_db.mview; --{serverError 60} -DROP DATABASE test_db; \ No newline at end of file +DROP DATABASE test_db; + diff --git a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql index 43c4efb8843..b733f42270a 100644 --- a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql +++ b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql @@ -24,4 +24,5 @@ INSERT INTO test_db.table VALUES (3,'some_val'); SELECT n,s FROM test_db.table ORDER BY n; SELECT n,n2 FROM test_db.mview ORDER by n; -DROP DATABASE test_db; \ No newline at end of file +DROP DATABASE test_db; + From cc92eacb3e4bdfb1292c465d32d533f9e6630330 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Fri, 24 Mar 2023 11:51:09 +0000 Subject: [PATCH 20/73] Merged several tests into one and rm unnecessary func --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 28 ++++++++----------- src/Interpreters/InterpreterCreateQuery.h | 1 - ...re_inacc_tables_mat_view_atttach.reference | 2 ++ ...6_ignore_inacc_tables_mat_view_atttach.sql | 21 ++++++++++---- ...re_inacc_tables_mat_view_atttach.reference | 0 ...t_ignore_inacc_tables_mat_view_atttach.sql | 21 -------------- ..._view_state_after_droping_tables.reference | 2 -- ...re_mat_view_state_after_droping_tables.sql | 28 ------------------- 9 files changed, 30 insertions(+), 75 deletions(-) delete mode 100644 tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference delete mode 100644 tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql delete mode 100644 tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference delete mode 100644 tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 644234248f1..6670920f613 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -467,7 +467,7 @@ class IColumn; M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ - M(Bool, ignore_inaccessible_tables_mat_view_attach, false, "Ignoring inaccessible tables for attaching to the materilized view.", 0) \ + M(Bool, skip_materialized_view_checking_if_source_table_not_exist, false, "Allow attaching to a materialized view even if dependent tables are inaccessible.", 0) \ \ M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 67dd053ef19..4e07a6ab017 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1077,22 +1077,6 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data } } -Block InterpreterCreateQuery::getSampleBlockFromCreteQuery(const ASTCreateQuery & create) -{ - Block result_block; - if (getContext()->getSettingsRef().allow_experimental_analyzer) - { - result_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); - } - else - { - result_block = InterpreterSelectWithUnionQuery(create.select->clone(), - getContext(), - SelectQueryOptions().analyze()).getSampleBlock(); - } - return result_block; -} - void InterpreterCreateQuery::checkTypecompatibleForMaterializeView(const ASTCreateQuery & create) { if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( @@ -1103,7 +1087,16 @@ void InterpreterCreateQuery::checkTypecompatibleForMaterializeView(const ASTCrea Block input_block; try { - input_block = getSampleBlockFromCreteQuery(create); + if (getContext()->getSettingsRef().allow_experimental_analyzer) + { + input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); + } + else + { + input_block = InterpreterSelectWithUnionQuery(create.select->clone(), + getContext(), + SelectQueryOptions().analyze()).getSampleBlock(); + } } catch (const Exception & e) { @@ -1112,6 +1105,7 @@ void InterpreterCreateQuery::checkTypecompatibleForMaterializeView(const ASTCrea ) { LOG_WARNING(&Poco::Logger::get("InterpreterSelectQuery"), "{}", e.message()); + return; } else { diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 0d3ce92a4ff..22e71715bcf 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -97,7 +97,6 @@ private: BlockIO fillTableIfNeeded(const ASTCreateQuery & create); /// Check type compatible for materialized dest table and select columns void checkTypecompatibleForMaterializeView(const ASTCreateQuery & create); - Block getSampleBlockFromCreteQuery(const ASTCreateQuery & create); void assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const; diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference index e69de29bb2d..bb3ee860aec 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.reference @@ -0,0 +1,2 @@ +3 some_val +3 9 diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index cfb363053e5..48f4bf2b2e2 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -1,8 +1,7 @@ -DROP DATABASE IF EXISTS test_db; - -SET skip_materialized_view_checking_if_source_table_not_exist = 1; SET send_logs_level = 'fatal'; +DROP DATABASE IF EXISTS test_db; + CREATE DATABASE test_db; CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; @@ -15,9 +14,21 @@ DROP TABLE test_db.table; DETACH TABLE test_db.mview; -ATTACH TABLE test_db.mview; +/* Check that we get an exception with the option. */ SET skip_materialized_view_checking_if_source_table_not_exist = 0; +ATTACH TABLE test_db.mview; --{serverError 60} + +/* Check that we don't get an exception with the option. */ +SET skip_materialized_view_checking_if_source_table_not_exist = 1; +ATTACH TABLE test_db.mview; + +/* Check if the data in the materialized view is updated after the restore.*/ +CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; + +INSERT INTO test_db.table VALUES (3,'some_val'); + +SELECT n,s FROM test_db.table ORDER BY n; +SELECT n,n2 FROM test_db.mview ORDER by n; DROP DATABASE test_db; - diff --git a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql deleted file mode 100644 index 492c2ed1fa2..00000000000 --- a/tests/queries/0_stateless/02697_do_not_ignore_inacc_tables_mat_view_atttach.sql +++ /dev/null @@ -1,21 +0,0 @@ -DROP DATABASE IF EXISTS test_db; - -SET skip_materialized_view_checking_if_source_table_not_exist = 0; -SET send_logs_level = 'fatal'; - -CREATE DATABASE test_db; - -CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; - -CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; - -CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; - -DROP TABLE test_db.table; - -DETACH TABLE test_db.mview; - -ATTACH TABLE test_db.mview; --{serverError 60} - -DROP DATABASE test_db; - diff --git a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference deleted file mode 100644 index bb3ee860aec..00000000000 --- a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.reference +++ /dev/null @@ -1,2 +0,0 @@ -3 some_val -3 9 diff --git a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql b/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql deleted file mode 100644 index b733f42270a..00000000000 --- a/tests/queries/0_stateless/02698_restore_mat_view_state_after_droping_tables.sql +++ /dev/null @@ -1,28 +0,0 @@ -DROP DATABASE IF EXISTS test_db; - -SET skip_materialized_view_checking_if_source_table_not_exist = 1; -SET send_logs_level = 'fatal'; - -CREATE DATABASE test_db; - -CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; - -CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; - -CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; - -DROP TABLE test_db.table; - -DETACH TABLE test_db.mview; - -ATTACH TABLE test_db.mview; - -CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; - -INSERT INTO test_db.table VALUES (3,'some_val'); - -SELECT n,s FROM test_db.table ORDER BY n; -SELECT n,n2 FROM test_db.mview ORDER by n; - -DROP DATABASE test_db; - From f353561204c5707fd229fcaeefc880096006cff2 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 14:37:40 +0000 Subject: [PATCH 21/73] Avoid breaking batches with read requests --- src/Coordination/KeeperDispatcher.cpp | 32 +++++++++++++++++++++---- src/Coordination/KeeperDispatcher.h | 16 +++++++++++++ src/Coordination/KeeperServer.cpp | 4 +++- src/Coordination/KeeperServer.h | 3 ++- src/Coordination/KeeperStateMachine.cpp | 5 +++- src/Coordination/KeeperStateMachine.h | 4 ++++ 6 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 2aa11dd9eed..fc97bc5e0e1 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -90,7 +90,7 @@ void KeeperDispatcher::requestThread() KeeperStorage::RequestsForSessions current_batch; - bool has_read_request = false; + bool has_read_request{false}; /// If new request is not read request or we must to process it through quorum. /// Otherwise we will process it locally. @@ -98,6 +98,8 @@ void KeeperDispatcher::requestThread() { current_batch.emplace_back(request); + size_t read_requests = 0; + const auto try_get_request = [&] { /// Trying to get batch requests as fast as possible @@ -106,7 +108,12 @@ void KeeperDispatcher::requestThread() CurrentMetrics::sub(CurrentMetrics::KeeperOutstandingRequets); /// Don't append read request into batch, we have to process them separately if (!coordination_settings->quorum_reads && request.request->isReadRequest()) - has_read_request = true; + { + ++read_requests; + std::pair key{current_batch.back().session_id, current_batch.back().request->xid}; + std::lock_guard lock(read_mutex); + related_read_requests[key].push_back(request); + } else current_batch.emplace_back(request); @@ -118,7 +125,7 @@ void KeeperDispatcher::requestThread() /// If we have enough requests in queue, we will try to batch at least max_quick_batch_size of them. size_t max_quick_batch_size = coordination_settings->max_requests_quick_batch_size; - while (!shutdown_called && !has_read_request && current_batch.size() < max_quick_batch_size && try_get_request()) + while (!shutdown_called && current_batch.size() + read_requests < max_quick_batch_size && try_get_request()) ; const auto prev_result_done = [&] @@ -129,7 +136,7 @@ void KeeperDispatcher::requestThread() }; /// Waiting until previous append will be successful, or batch is big enough - while (!shutdown_called && !has_read_request && !prev_result_done() && current_batch.size() <= max_batch_size) + while (!shutdown_called && !prev_result_done() && current_batch.size() <= max_batch_size) { try_get_request(); } @@ -319,7 +326,22 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf snapshot_s3.startup(config, macros); - server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3); + server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3, [this](const KeeperStorage::RequestForSession & request_for_session) + { + std::lock_guard lock(read_mutex); + if (auto it = related_read_requests.find(std::pair{request_for_session.session_id, request_for_session.request->xid}); it != related_read_requests.end()) + { + for (const auto & read_request : it->second) + { + if (server->isLeaderAlive()) + server->putLocalReadRequest(read_request); + else + addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); + } + + related_read_requests.erase(it); + } + }); try { diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index 9371d2fbbac..e7570727b9a 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -1,5 +1,6 @@ #pragma once +#include "Common/ZooKeeper/ZooKeeperCommon.h" #include "config.h" #if USE_NURAFT @@ -103,6 +104,21 @@ private: void forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions); public: + std::mutex read_mutex; + + struct PairHash + { + auto operator()(std::pair pair) const + { + SipHash hash; + hash.update(pair.first); + hash.update(pair.second); + return hash.get64(); + } + }; + + std::unordered_map, KeeperStorage::RequestsForSessions, PairHash> related_read_requests; + /// Just allocate some objects, real initialization is done by `intialize method` KeeperDispatcher(); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 78a095f8c8d..56ed8f4eafe 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -107,7 +107,8 @@ KeeperServer::KeeperServer( const Poco::Util::AbstractConfiguration & config, ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, - KeeperSnapshotManagerS3 & snapshot_manager_s3) + KeeperSnapshotManagerS3 & snapshot_manager_s3, + KeeperStateMachine::CommitCallback commit_callback) : server_id(configuration_and_settings_->server_id) , coordination_settings(configuration_and_settings_->coordination_settings) , log(&Poco::Logger::get("KeeperServer")) @@ -128,6 +129,7 @@ KeeperServer::KeeperServer( coordination_settings, keeper_context, config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr, + commit_callback, checkAndGetSuperdigest(configuration_and_settings_->super_digest)); state_manager = nuraft::cs_new( diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index bcff81f66f2..db4e9c1962e 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -72,7 +72,8 @@ public: const Poco::Util::AbstractConfiguration & config_, ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, - KeeperSnapshotManagerS3 & snapshot_manager_s3); + KeeperSnapshotManagerS3 & snapshot_manager_s3, + KeeperStateMachine::CommitCallback commit_callback); /// Load state machine from the latest snapshot and load log storage. Start NuRaft with required settings. void startup(const Poco::Util::AbstractConfiguration & config, bool enable_ipv6 = true); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index d0bd18b63e2..0b69b00bf0e 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -46,8 +46,10 @@ KeeperStateMachine::KeeperStateMachine( const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, + CommitCallback commit_callback_, const std::string & superdigest_) - : coordination_settings(coordination_settings_) + : commit_callback(commit_callback_) + , coordination_settings(coordination_settings_) , snapshot_manager( snapshots_path_, coordination_settings->snapshots_to_keep, @@ -274,6 +276,7 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n ProfileEvents::increment(ProfileEvents::KeeperCommits); last_committed_idx = log_idx; + commit_callback(request_for_session); return nullptr; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index d8181532f09..6babf741dbd 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -22,6 +22,8 @@ using SnapshotsQueue = ConcurrentBoundedQueue; class KeeperStateMachine : public nuraft::state_machine { public: + using CommitCallback = std::function; + KeeperStateMachine( ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, @@ -29,6 +31,7 @@ public: const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, + CommitCallback commit_callback_, const std::string & superdigest_ = ""); /// Read state from the latest snapshot @@ -105,6 +108,7 @@ public: void recalculateStorageStats(); private: + CommitCallback commit_callback; /// In our state machine we always have a single snapshot which is stored /// in memory in compressed (serialized) format. SnapshotMetadataPtr latest_snapshot_meta = nullptr; From f7c0cca297c0e093fe24eaa225f125c2f0cfd31e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 14:37:51 +0000 Subject: [PATCH 22/73] Maybe better keeper-bench --- utils/keeper-bench/Generator.cpp | 21 ++++++++++++++++++++- utils/keeper-bench/Generator.h | 14 ++++++++++++++ utils/keeper-bench/Runner.cpp | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 5d1d0f8a491..b6d8223862c 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -162,6 +162,19 @@ ZooKeeperRequestPtr SetRequestGenerator::generate() return request; } +void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +{ + for (auto & generator : generators) + generator->startup(zookeeper); +} + +ZooKeeperRequestPtr MixedRequestGenerator::generate() +{ + pcg64 rng(randomSeed()); + std::uniform_int_distribution distribution(0, generators.size() - 1); + + return generators[distribution(rng)]->generate(); +} void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) { @@ -315,7 +328,13 @@ std::unique_ptr getGenerator(const std::string & name) { return std::make_unique("/set_generator", 5); } - + else if (name == "mixed_small_data") + { + std::vector> generators; + generators.push_back(std::make_unique("/set_generator", 5)); + generators.push_back(std::make_unique("/get_generator", 10, 32)); + return std::make_unique(std::move(generators)); + } throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 1ff01b25ed4..e2c546e4bce 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -122,5 +122,19 @@ private: uint64_t data_size; }; +class MixedRequestGenerator final : public IGenerator +{ +public: + explicit MixedRequestGenerator(std::vector> generators_) + : generators(std::move(generators_)) + {} + + void startup(Coordination::ZooKeeper & zookeeper) override; + Coordination::ZooKeeperRequestPtr generate() override; + +private: + std::vector> generators; +}; + std::unique_ptr getGenerator(const std::string & name); diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 2f3cf4b0620..c858b476483 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -159,9 +159,9 @@ void Runner::runBenchmark() std::cerr << "Prepared\n"; try { + auto connections = getConnections(); for (size_t i = 0; i < concurrency; ++i) { - auto connections = getConnections(); pool.scheduleOrThrowOnError([this, connections]() mutable { thread(connections); }); } } From ef4f860b62078e7fa19167209449f80e2448cf28 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 24 Mar 2023 11:51:22 -0300 Subject: [PATCH 23/73] namespace fix --- .../tests/gtest_assert_arrow_log_does_not_abort.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp index fb13f4664d9..203f2d7cef6 100644 --- a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp +++ b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp @@ -3,7 +3,8 @@ #include #include -using namespace DB; +namespace DB +{ TEST(ChunkedArray, ChunkedArrayWithZeroChunksShouldNotAbort) { @@ -16,3 +17,5 @@ TEST(ArrowLog, FatalLogShouldThrow) { EXPECT_ANY_THROW(::arrow::util::ArrowLog(__FILE__, __LINE__, ::arrow::util::ArrowLogLevel::ARROW_FATAL)); } + +} From 7a82830f1086548f348b398b6e2c33adbb12ba9c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Mar 2023 15:04:05 +0000 Subject: [PATCH 24/73] Automatic style fix --- docker/test/performance-comparison/perf.py | 2 ++ docker/test/performance-comparison/report.py | 1 - tests/ci/clickhouse_helper.py | 1 - tests/ci/docker_images_check.py | 1 - tests/ci/get_previous_release_tag.py | 1 - tests/ci/report.py | 2 +- tests/integration/helpers/cluster.py | 3 +-- tests/integration/helpers/network.py | 2 -- .../pytest_xdist_logging_to_separate_files.py | 1 + .../test_detach_part_wrong_partition_id.py | 1 - .../test_cluster_copier/test_three_nodes.py | 1 - .../test_cluster_copier/test_two_nodes.py | 1 - tests/integration/test_composable_protocols/test.py | 1 - .../test_create_query_constraints/test.py | 2 -- .../common.py | 1 - tests/integration/test_disks_app_func/test.py | 1 - .../test_distributed_ddl_parallel/test.py | 1 + tests/integration/test_fetch_memory_usage/test.py | 1 - .../scripts/stress_test.py | 1 - tests/integration/test_jbod_balancer/test.py | 1 - .../test_keeper_and_access_storage/test.py | 1 + tests/integration/test_keeper_back_to_back/test.py | 2 +- tests/integration/test_keeper_persistent_log/test.py | 1 - .../test_keeper_zookeeper_converter/test.py | 1 - tests/integration/test_merge_tree_load_parts/test.py | 6 +++--- .../s3_endpoint/endpoint.py | 1 - .../test_merge_tree_settings_constraints/test.py | 1 - .../test_old_parts_finally_removed/test.py | 1 - tests/integration/test_partition/test.py | 4 +++- tests/integration/test_password_constraints/test.py | 1 - tests/integration/test_read_only_table/test.py | 1 - .../test_reload_auxiliary_zookeepers/test.py | 1 - .../s3_endpoint/endpoint.py | 1 + tests/integration/test_s3_with_proxy/test.py | 1 + .../integration/test_ssl_cert_authentication/test.py | 1 - tests/integration/test_storage_kafka/kafka_pb2.py | 1 - .../test_storage_kafka/message_with_repeated_pb2.py | 1 - tests/integration/test_storage_kafka/social_pb2.py | 1 - tests/integration/test_storage_kafka/test.py | 12 ++---------- tests/integration/test_storage_nats/nats_pb2.py | 1 - .../test_storage_postgresql_replica/test.py | 1 - .../test_storage_rabbitmq/rabbitmq_pb2.py | 1 - tests/integration/test_storage_rabbitmq/test.py | 3 --- tests/integration/test_storage_s3/test.py | 1 + .../test_storage_s3/test_invalid_env_credentials.py | 1 + tests/integration/test_system_merges/test.py | 1 - tests/integration/test_ttl_move/test.py | 2 +- tests/integration/test_zero_copy_fetch/test.py | 1 - utils/changelog-simple/format-changelog.py | 1 + utils/keeper-overload/keeper-overload.py | 2 +- 50 files changed, 23 insertions(+), 57 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 65bf49c2914..7a4e6386d0d 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -26,6 +26,7 @@ logging.basicConfig( total_start_seconds = time.perf_counter() stage_start_seconds = total_start_seconds + # Thread executor that does not hides exception that happens during function # execution, and rethrows it after join() class SafeThread(Thread): @@ -158,6 +159,7 @@ for e in subst_elems: available_parameters[name] = values + # Takes parallel lists of templates, substitutes them with all combos of # parameters. The set of parameters is determined based on the first list. # Note: keep the order of queries -- sometimes we have DROP IF EXISTS diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 782cf29863c..214f2d550b4 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -670,7 +670,6 @@ if args.report == "main": ) elif args.report == "all-queries": - print((header_template.format())) add_tested_commits() diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index d60a9e6afd1..64b64896f66 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -141,7 +141,6 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" base_ref = "master" head_ref = "master" diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 192d216614e..f2b1105b3b0 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -96,7 +96,6 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( pr_info: PRInfo, images_dict: ImagesDict ) -> Set[DockerImage]: - if not images_dict: return set() diff --git a/tests/ci/get_previous_release_tag.py b/tests/ci/get_previous_release_tag.py index c6fe6cd5fb5..c2d279f7fec 100755 --- a/tests/ci/get_previous_release_tag.py +++ b/tests/ci/get_previous_release_tag.py @@ -51,7 +51,6 @@ def find_previous_release( for release in releases: if release.version < server_version: - # Check if the artifact exists on GitHub. # It can be not true for a short period of time # after creating a tag for a new release before uploading the packages. diff --git a/tests/ci/report.py b/tests/ci/report.py index 947fb33d905..ddee035d26f 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -473,7 +473,7 @@ def create_build_html_report( commit_url: str, ) -> str: rows = "" - for (build_result, build_log_url, artifact_urls) in zip( + for build_result, build_log_url, artifact_urls in zip( build_results, build_logs_urls, artifact_urls_list ): row = "" diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index dc5ada81995..a9a996e0a5f 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -63,6 +63,7 @@ DEFAULT_ENV_NAME = ".env" SANITIZER_SIGN = "==================" + # to create docker-compose env file def _create_env_file(path, variables): logging.debug(f"Env {variables} stored in {path}") @@ -1454,7 +1455,6 @@ class ClickHouseCluster: config_root_name="clickhouse", extra_configs=[], ) -> "ClickHouseInstance": - """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -3089,7 +3089,6 @@ class ClickHouseInstance: config_root_name="clickhouse", extra_configs=[], ): - self.name = name self.base_cmd = cluster.base_cmd self.docker_id = cluster.get_instance_docker_id(self.name) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e408c9beec1..471aa2bdc2e 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -216,7 +216,6 @@ class _NetworkManager: container_exit_timeout=60, docker_api_version=os.environ.get("DOCKER_API_VERSION"), ): - self.container_expire_timeout = container_expire_timeout self.container_exit_timeout = container_exit_timeout @@ -232,7 +231,6 @@ class _NetworkManager: def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): - for i in range(5): if self._container is not None: try: diff --git a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py index d424ad58fa4..370aa23a014 100644 --- a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py +++ b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py @@ -1,6 +1,7 @@ import logging import os.path + # Makes the parallel workers of pytest-xdist to log to separate files. # Without this function all workers will log to the same log file # and mix everything together making it much more difficult for troubleshooting. diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py index 02fccfae4e5..a6f7a8653da 100644 --- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -24,7 +24,6 @@ def start_cluster(): def test_detach_part_wrong_partition_id(start_cluster): - # Here we create table with partition by UUID. node_21_6.query( "create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()" diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index 31d6c0448f4..e7d07757adb 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -19,7 +19,6 @@ cluster = ClickHouseCluster(__file__) def started_cluster(): global cluster try: - for name in ["first", "second", "third"]: cluster.add_instance( name, diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py index 10ab7d03b00..2b6fcf6cac2 100644 --- a/tests/integration/test_cluster_copier/test_two_nodes.py +++ b/tests/integration/test_cluster_copier/test_two_nodes.py @@ -19,7 +19,6 @@ cluster = ClickHouseCluster(__file__) def started_cluster(): global cluster try: - for name in ["first_of_two", "second_of_two"]: instance = cluster.add_instance( name, diff --git a/tests/integration/test_composable_protocols/test.py b/tests/integration/test_composable_protocols/test.py index bc87fea5296..df74cfffa54 100644 --- a/tests/integration/test_composable_protocols/test.py +++ b/tests/integration/test_composable_protocols/test.py @@ -63,7 +63,6 @@ def netcat(hostname, port, content): def test_connections(): - client = Client(server.ip_address, 9000, command=cluster.client_bin_path) assert client.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_create_query_constraints/test.py b/tests/integration/test_create_query_constraints/test.py index 8df043fd24b..33c41b4f161 100644 --- a/tests/integration/test_create_query_constraints/test.py +++ b/tests/integration/test_create_query_constraints/test.py @@ -25,7 +25,6 @@ def start_cluster(): def test_create_query_const_constraints(): - instance.query("CREATE USER u_const SETTINGS max_threads = 1 CONST") instance.query("GRANT ALL ON *.* TO u_const") @@ -57,7 +56,6 @@ def test_create_query_const_constraints(): def test_create_query_minmax_constraints(): - instance.query("CREATE USER u_minmax SETTINGS max_threads = 4 MIN 2 MAX 6") instance.query("GRANT ALL ON *.* TO u_minmax") diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py index b38e81b0227..01addae2542 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py @@ -348,7 +348,6 @@ class RangedLayoutTester(BaseLayoutTester): self.layouts = LAYOUTS_RANGED def execute(self, layout_name, node): - if layout_name not in self.layout_to_dictionary: raise RuntimeError("Source doesn't support layout: {}".format(layout_name)) diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 027ef8feed0..2428c53854e 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -7,7 +7,6 @@ import pytest def started_cluster(): global cluster try: - cluster = ClickHouseCluster(__file__) cluster.add_instance( "disks_app_test", main_configs=["config.xml"], with_minio=True diff --git a/tests/integration/test_distributed_ddl_parallel/test.py b/tests/integration/test_distributed_ddl_parallel/test.py index 6ebfe472e09..eb98dd3e230 100644 --- a/tests/integration/test_distributed_ddl_parallel/test.py +++ b/tests/integration/test_distributed_ddl_parallel/test.py @@ -10,6 +10,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) + # By default the exceptions that was throwed in threads will be ignored # (they will not mark the test as failed, only printed to stderr). # diff --git a/tests/integration/test_fetch_memory_usage/test.py b/tests/integration/test_fetch_memory_usage/test.py index a4371140150..7591cc0e8a9 100644 --- a/tests/integration/test_fetch_memory_usage/test.py +++ b/tests/integration/test_fetch_memory_usage/test.py @@ -18,7 +18,6 @@ def started_cluster(): def test_huge_column(started_cluster): - if ( node.is_built_with_thread_sanitizer() or node.is_built_with_memory_sanitizer() diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py index b8bafb3d0c1..fe69d72c1c7 100644 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py +++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py @@ -13,7 +13,6 @@ number_of_iterations = 100 def perform_request(): - buffer = BytesIO() crl = pycurl.Curl() crl.setopt(pycurl.INTERFACE, client_ip) diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index e746698611a..df34a075d5a 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -45,7 +45,6 @@ def start_cluster(): def check_balance(node, table): - partitions = node.query( """ WITH diff --git a/tests/integration/test_keeper_and_access_storage/test.py b/tests/integration/test_keeper_and_access_storage/test.py index 6ec307f7082..0314825b6b7 100644 --- a/tests/integration/test_keeper_and_access_storage/test.py +++ b/tests/integration/test_keeper_and_access_storage/test.py @@ -10,6 +10,7 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/keeper.xml"], stay_alive=True ) + # test that server is able to start @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 73fface02b4..b737ac284d2 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -546,7 +546,6 @@ def test_random_requests(started_cluster): def test_end_of_session(started_cluster): - fake_zk1 = None fake_zk2 = None genuine_zk1 = None @@ -685,6 +684,7 @@ def test_concurrent_watches(started_cluster): nonlocal watches_created nonlocal all_paths_created fake_zk.ensure_path(global_path + "/" + str(i)) + # new function each time def dumb_watch(event): nonlocal dumb_watch_triggered_counter diff --git a/tests/integration/test_keeper_persistent_log/test.py b/tests/integration/test_keeper_persistent_log/test.py index 70cc14fe26d..4164ffb33d3 100644 --- a/tests/integration/test_keeper_persistent_log/test.py +++ b/tests/integration/test_keeper_persistent_log/test.py @@ -163,7 +163,6 @@ def test_state_duplicate_restart(started_cluster): # http://zookeeper-user.578899.n2.nabble.com/Why-are-ephemeral-nodes-written-to-disk-tp7583403p7583418.html def test_ephemeral_after_restart(started_cluster): - try: node_zk = None node_zk2 = None diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index 063421bf922..de5a9416119 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -114,7 +114,6 @@ def start_clickhouse(): def copy_zookeeper_data(make_zk_snapshots): - if make_zk_snapshots: # force zookeeper to create snapshot generate_zk_snapshot() else: diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py index 777b6f14fc6..dfbe00c8e28 100644 --- a/tests/integration/test_merge_tree_load_parts/test.py +++ b/tests/integration/test_merge_tree_load_parts/test.py @@ -148,17 +148,17 @@ def test_merge_tree_load_parts_corrupted(started_cluster): node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts_2") def check_parts_loading(node, partition, loaded, failed, skipped): - for (min_block, max_block) in loaded: + for min_block, max_block in loaded: part_name = f"{partition}_{min_block}_{max_block}" assert node.contains_in_log(f"Loading Active part {part_name}") assert node.contains_in_log(f"Finished loading Active part {part_name}") - for (min_block, max_block) in failed: + for min_block, max_block in failed: part_name = f"{partition}_{min_block}_{max_block}" assert node.contains_in_log(f"Loading Active part {part_name}") assert not node.contains_in_log(f"Finished loading Active part {part_name}") - for (min_block, max_block) in skipped: + for min_block, max_block in skipped: part_name = f"{partition}_{min_block}_{max_block}" assert not node.contains_in_log(f"Loading Active part {part_name}") assert not node.contains_in_log(f"Finished loading Active part {part_name}") diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index b6567dfebc5..4613fdb850b 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -42,7 +42,6 @@ def delete(_bucket): @route("/<_bucket>/<_path:path>", ["GET", "POST", "PUT", "DELETE"]) def server(_bucket, _path): - # It's delete query for failed part if _path.endswith("delete"): response.set_header("Location", "http://minio1:9001/" + _bucket + "/" + _path) diff --git a/tests/integration/test_merge_tree_settings_constraints/test.py b/tests/integration/test_merge_tree_settings_constraints/test.py index 0bb0179108d..be6e2a31873 100644 --- a/tests/integration/test_merge_tree_settings_constraints/test.py +++ b/tests/integration/test_merge_tree_settings_constraints/test.py @@ -20,7 +20,6 @@ def start_cluster(): def test_merge_tree_settings_constraints(): - assert "Setting storage_policy should not be changed" in instance.query_and_get_error( f"CREATE TABLE wrong_table (number Int64) engine = MergeTree() ORDER BY number SETTINGS storage_policy = 'secret_policy'" ) diff --git a/tests/integration/test_old_parts_finally_removed/test.py b/tests/integration/test_old_parts_finally_removed/test.py index 108b72c5ccd..5347d433419 100644 --- a/tests/integration/test_old_parts_finally_removed/test.py +++ b/tests/integration/test_old_parts_finally_removed/test.py @@ -63,7 +63,6 @@ def test_part_finally_removed(started_cluster): ) for i in range(60): - if ( node1.query( "SELECT count() from system.parts WHERE table = 'drop_outdated_part'" diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index ae4393fc6f6..a34141c6189 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -528,7 +528,9 @@ def test_make_clone_in_detached(started_cluster): ["cp", "-r", path + "all_0_0_0", path + "detached/broken_all_0_0_0"] ) assert_eq_with_retry(instance, "select * from clone_in_detached", "\n") - assert ["broken_all_0_0_0",] == sorted( + assert [ + "broken_all_0_0_0", + ] == sorted( instance.exec_in_container(["ls", path + "detached/"]).strip().split("\n") ) diff --git a/tests/integration/test_password_constraints/test.py b/tests/integration/test_password_constraints/test.py index e3628861b28..9cdff51caa1 100644 --- a/tests/integration/test_password_constraints/test.py +++ b/tests/integration/test_password_constraints/test.py @@ -17,7 +17,6 @@ def start_cluster(): def test_complexity_rules(start_cluster): - error_message = "DB::Exception: Invalid password. The password should: be at least 12 characters long, contain at least 1 numeric character, contain at least 1 lowercase character, contain at least 1 uppercase character, contain at least 1 special character" assert error_message in node.query_and_get_error( "CREATE USER u_1 IDENTIFIED WITH plaintext_password BY ''" diff --git a/tests/integration/test_read_only_table/test.py b/tests/integration/test_read_only_table/test.py index 914c6a99508..df084f9dbbd 100644 --- a/tests/integration/test_read_only_table/test.py +++ b/tests/integration/test_read_only_table/test.py @@ -49,7 +49,6 @@ def start_cluster(): def test_restart_zookeeper(start_cluster): - for table_id in range(NUM_TABLES): node1.query( f"INSERT INTO test_table_{table_id} VALUES (1), (2), (3), (4), (5);" diff --git a/tests/integration/test_reload_auxiliary_zookeepers/test.py b/tests/integration/test_reload_auxiliary_zookeepers/test.py index bb1455333fc..476c5dee99e 100644 --- a/tests/integration/test_reload_auxiliary_zookeepers/test.py +++ b/tests/integration/test_reload_auxiliary_zookeepers/test.py @@ -20,7 +20,6 @@ def start_cluster(): def test_reload_auxiliary_zookeepers(start_cluster): - node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py index d6a732cc681..1d33ca02f86 100644 --- a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from bottle import request, route, run, response + # Handle for MultipleObjectsDelete. @route("/<_bucket>", ["POST"]) def delete(_bucket): diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 1102d190a87..1af040c3c30 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -5,6 +5,7 @@ import time import pytest from helpers.cluster import ClickHouseCluster + # Runs simple proxy resolver in python env container. def run_resolver(cluster): container_id = cluster.get_container_id("resolver") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 7c62ca0d8b6..b3570b6e281 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -87,7 +87,6 @@ config = """ def execute_query_native(node, query, user, cert_name): - config_path = f"{SCRIPT_DIR}/configs/client.xml" formatted = config.format( diff --git a/tests/integration/test_storage_kafka/kafka_pb2.py b/tests/integration/test_storage_kafka/kafka_pb2.py index 7de1363bbf1..3e47af6c1e0 100644 --- a/tests/integration/test_storage_kafka/kafka_pb2.py +++ b/tests/integration/test_storage_kafka/kafka_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.kafka_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _KEYVALUEPAIR._serialized_start = 46 _KEYVALUEPAIR._serialized_end = 88 diff --git a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py index 4d1a23c0b43..3715a9bea04 100644 --- a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py +++ b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.message_with_repeated_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"H\001" _MESSAGE._serialized_start = 62 diff --git a/tests/integration/test_storage_kafka/social_pb2.py b/tests/integration/test_storage_kafka/social_pb2.py index 830ade81d33..f91a7bd0539 100644 --- a/tests/integration/test_storage_kafka/social_pb2.py +++ b/tests/integration/test_storage_kafka/social_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.social_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _USER._serialized_start = 47 _USER._serialized_end = 90 diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 51952ac1eb7..3a4fa6c6bfe 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -121,7 +121,7 @@ def kafka_create_topic( def kafka_delete_topic(admin_client, topic, max_retries=50): result = admin_client.delete_topics([topic]) - for (topic, e) in result.topic_error_codes: + for topic, e in result.topic_error_codes: if e == 0: logging.debug(f"Topic {topic} deleted") else: @@ -917,9 +917,7 @@ def describe_consumer_group(kafka_cluster, name): member_info["client_id"] = client_id member_info["client_host"] = client_host member_topics_assignment = [] - for (topic, partitions) in MemberAssignment.decode( - member_assignment - ).assignment: + for topic, partitions in MemberAssignment.decode(member_assignment).assignment: member_topics_assignment.append({"topic": topic, "partitions": partitions}) member_info["assignment"] = member_topics_assignment res.append(member_info) @@ -1537,7 +1535,6 @@ def test_kafka_protobuf_no_delimiter(kafka_cluster): def test_kafka_materialized_view(kafka_cluster): - instance.query( """ DROP TABLE IF EXISTS test.view; @@ -2315,7 +2312,6 @@ def test_kafka_virtual_columns2(kafka_cluster): def test_kafka_produce_key_timestamp(kafka_cluster): - admin_client = KafkaAdminClient( bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) @@ -2444,7 +2440,6 @@ def test_kafka_insert_avro(kafka_cluster): def test_kafka_produce_consume_avro(kafka_cluster): - admin_client = KafkaAdminClient( bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) @@ -4031,7 +4026,6 @@ def test_kafka_predefined_configuration(kafka_cluster): # https://github.com/ClickHouse/ClickHouse/issues/26643 def test_issue26643(kafka_cluster): - # for backporting: # admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") admin_client = KafkaAdminClient( @@ -4313,7 +4307,6 @@ def test_row_based_formats(kafka_cluster): "RowBinaryWithNamesAndTypes", "MsgPack", ]: - print(format_name) kafka_create_topic(admin_client, format_name) @@ -4438,7 +4431,6 @@ def test_block_based_formats_2(kafka_cluster): "ORC", "JSONCompactColumns", ]: - kafka_create_topic(admin_client, format_name) instance.query( diff --git a/tests/integration/test_storage_nats/nats_pb2.py b/tests/integration/test_storage_nats/nats_pb2.py index 4330ff57950..e9e5cb72363 100644 --- a/tests/integration/test_storage_nats/nats_pb2.py +++ b/tests/integration/test_storage_nats/nats_pb2.py @@ -31,7 +31,6 @@ ProtoKeyValue = _reflection.GeneratedProtocolMessageType( _sym_db.RegisterMessage(ProtoKeyValue) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _PROTOKEYVALUE._serialized_start = 45 _PROTOKEYVALUE._serialized_end = 88 diff --git a/tests/integration/test_storage_postgresql_replica/test.py b/tests/integration/test_storage_postgresql_replica/test.py index 5df8b9029e6..8666d7ae58c 100644 --- a/tests/integration/test_storage_postgresql_replica/test.py +++ b/tests/integration/test_storage_postgresql_replica/test.py @@ -706,7 +706,6 @@ def test_abrupt_connection_loss_while_heavy_replication(started_cluster): def test_abrupt_server_restart_while_heavy_replication(started_cluster): - # FIXME (kssenii) temporary disabled if instance.is_built_with_sanitizer(): pytest.skip("Temporary disabled (FIXME)") diff --git a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py index e017b4e66c2..a5845652eef 100644 --- a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py +++ b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.rabbitmq_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _KEYVALUEPROTO._serialized_start = 49 _KEYVALUEPROTO._serialized_end = 92 diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 2e54f21787a..53b6c4109ef 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2864,7 +2864,6 @@ def test_rabbitmq_predefined_configuration(rabbitmq_cluster): def test_rabbitmq_msgpack(rabbitmq_cluster): - instance.query( """ drop table if exists rabbit_in; @@ -2908,7 +2907,6 @@ def test_rabbitmq_msgpack(rabbitmq_cluster): def test_rabbitmq_address(rabbitmq_cluster): - instance2.query( """ drop table if exists rabbit_in; @@ -3243,7 +3241,6 @@ def test_block_based_formats_2(rabbitmq_cluster): "ORC", "JSONCompactColumns", ]: - print(format_name) instance.query( diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 8b20727a7b5..4d493d9526b 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -18,6 +18,7 @@ MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(started_cluster): # Allows read-write access for bucket without authorization. diff --git a/tests/integration/test_storage_s3/test_invalid_env_credentials.py b/tests/integration/test_storage_s3/test_invalid_env_credentials.py index 2f5d9349904..aa6479a2ed3 100644 --- a/tests/integration/test_storage_s3/test_invalid_env_credentials.py +++ b/tests/integration/test_storage_s3/test_invalid_env_credentials.py @@ -11,6 +11,7 @@ MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(started_cluster): # Allows read-write access for bucket without authorization. diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 0a469bd7bbd..ff303afe19e 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -171,7 +171,6 @@ def test_mutation_simple(started_cluster, replicated): starting_block = 0 if replicated else 1 try: - for node in nodes: node.query( f"create table {name} (a Int64) engine={engine} order by tuple()" diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 99978cbf6dc..89824293320 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1863,7 +1863,7 @@ def test_ttl_move_if_exists(started_cluster, name, dest_type): ) ) - for (node, policy) in zip( + for node, policy in zip( [node1, node2], ["only_jbod_1", "small_jbod_with_external"] ): node.query( diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py index b71752528d3..9b9aa5e0da7 100644 --- a/tests/integration/test_zero_copy_fetch/test.py +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -16,7 +16,6 @@ cluster = ClickHouseCluster(__file__) @pytest.fixture(scope="module") def started_cluster(): try: - cluster.add_instance( "node1", main_configs=["configs/storage_conf.xml"], diff --git a/utils/changelog-simple/format-changelog.py b/utils/changelog-simple/format-changelog.py index d5e1518270e..01f2694dd0f 100755 --- a/utils/changelog-simple/format-changelog.py +++ b/utils/changelog-simple/format-changelog.py @@ -20,6 +20,7 @@ parser.add_argument( ) args = parser.parse_args() + # This function mirrors the PR description checks in ClickhousePullRequestTrigger. # Returns False if the PR should not be mentioned changelog. def parse_one_pull_request(item): diff --git a/utils/keeper-overload/keeper-overload.py b/utils/keeper-overload/keeper-overload.py index bdb4563c713..0a059b10588 100755 --- a/utils/keeper-overload/keeper-overload.py +++ b/utils/keeper-overload/keeper-overload.py @@ -166,7 +166,7 @@ def main(args): keeper_bench_path = args.keeper_bench_path keepers = [] - for (port, server_id) in zip(PORTS, SERVER_IDS): + for port, server_id in zip(PORTS, SERVER_IDS): keepers.append( Keeper( keeper_binary_path, server_id, port, workdir, args.with_thread_fuzzer From ee0fd39f2a1959a792977a114523bdd477c7ed16 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 24 Mar 2023 15:20:19 +0000 Subject: [PATCH 25/73] Fix data-race --- src/Interpreters/ProcessList.cpp | 10 ++++++++-- src/Interpreters/ProcessList.h | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index c7916f3eed2..d7ba0b8330f 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -388,6 +388,12 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } +bool QueryStatus::ExecutorHolder::equals(const PipelineExecutor * e) +{ + std::lock_guard lock(mutex); + return executor == e; +} + CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -428,7 +434,7 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }) == executors.end()); + assert(std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }) == executors.end()); executors.push_back(std::make_shared(e)); } @@ -437,7 +443,7 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) ExecutorHolderPtr executor_holder; { std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + auto it = std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }); assert(it != executors.end()); executor_holder = *it; } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 30bfde4e218..98d7daa0949 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -127,6 +127,8 @@ protected: void remove(); + bool equals(const PipelineExecutor * e); + PipelineExecutor * executor; std::mutex mutex; }; From 16ebfcc3f8b080968107751b46a7150c13d5680f Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Mar 2023 16:15:52 +0100 Subject: [PATCH 26/73] Better delegate disks handle mutual path --- src/Disks/DiskEncrypted.h | 4 +++ src/Disks/IDisk.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 33 +++++++++---------- .../test_encrypted_disk/configs/storage.xml | 1 - tests/integration/test_encrypted_disk/test.py | 23 +++++++++++++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h index d38c916ee6e..d37e3a560bc 100644 --- a/src/Disks/DiskEncrypted.h +++ b/src/Disks/DiskEncrypted.h @@ -276,6 +276,10 @@ public: return delegate->getMetadataStorage(); } + DiskPtr getDelegateDiskIfExists() const override + { + return delegate; + } private: String wrappedPath(const String & path) const diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index ea117c0e0c6..4e488bbb39a 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -423,6 +423,8 @@ public: void markDiskAsCustom() { is_custom_disk = true; } + virtual DiskPtr getDelegateDiskIfExists() const { return nullptr; } + protected: friend class DiskDecorator; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 84c301e5986..f7814fcf276 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1517,24 +1517,26 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { /// Check extra parts at different disks, in order to not allow to miss data parts at undefined disks. std::unordered_set defined_disk_names; - /// If disk is wrapped into cached disk, it will not be defined in storage policy. - std::unordered_set disk_names_wrapped_in_cache; for (const auto & disk_ptr : disks) + { defined_disk_names.insert(disk_ptr->getName()); - for (const auto & [disk_name, disk_ptr] : getContext()->getDisksMap()) - { - /// In composable cache with the underlying source disk there might the following structure: + /// As encrypted disk can use the same path of its nested disk, + /// we need to take it into account here. + const auto & delegate = disk_ptr->getDelegateDiskIfExists(); + if (delegate && disk_ptr->getPath() == delegate->getPath()) + defined_disk_names.insert(delegate->getName()); + + /// As cache is implemented on object storage layer, not on disk level, e.g. + /// we have such structure: /// DiskObjectStorage(CachedObjectStorage(...(CachedObjectStored(ObjectStorage)...))) - /// In configuration file each of these layers has a different name, but data path - /// (getPath() result) is the same. We need to take it into account here. - if (disk_ptr->supportsCache() && defined_disk_names.contains(disk_ptr->getName())) + /// and disk_ptr->getName() here is the name of last delegate - ObjectStorage. + /// So now we need to add cache layers to defined disk names. + if (disk_ptr->supportsCache()) { auto caches = disk_ptr->getCacheLayersNames(); - disk_names_wrapped_in_cache.insert(caches.begin(), caches.end()); - LOG_TEST(log, "Cache layers for cache disk `{}`, inner disk `{}`: {}", - disk_name, disk_ptr->getName(), fmt::join(caches, ", ")); + defined_disk_names.insert(caches.begin(), caches.end()); } } @@ -1543,9 +1545,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (disk->isBroken() || disk->isCustomDisk()) continue; - if (!defined_disk_names.contains(disk_name) - && disk->exists(relative_data_path) - && !disk_names_wrapped_in_cache.contains(disk_name)) + if (!defined_disk_names.contains(disk_name) && disk->exists(relative_data_path)) { for (const auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) { @@ -1553,9 +1553,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { throw Exception( ErrorCodes::UNKNOWN_DISK, - "Part {} ({}) was found on disk {} which is not defined in the storage policy (defined disks: {}, wrapped disks: {})", - backQuote(it->name()), backQuote(it->path()), backQuote(disk_name), - fmt::join(defined_disk_names, ", "), fmt::join(disk_names_wrapped_in_cache, ", ")); + "Part {} ({}) was found on disk {} which is not defined in the storage policy (defined disks: {})", + backQuote(it->name()), backQuote(it->path()), backQuote(disk_name), fmt::join(defined_disk_names, ", ")); } } } diff --git a/tests/integration/test_encrypted_disk/configs/storage.xml b/tests/integration/test_encrypted_disk/configs/storage.xml index 5cba95a89a0..a6fff813f2e 100644 --- a/tests/integration/test_encrypted_disk/configs/storage.xml +++ b/tests/integration/test_encrypted_disk/configs/storage.xml @@ -14,7 +14,6 @@ encrypted disk_s3 - encrypted/ 1234567812345678 diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 42ce7aab906..5b85888201b 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -12,6 +12,7 @@ node = cluster.add_instance( main_configs=["configs/storage.xml"], tmpfs=["/disk:size=100M"], with_minio=True, + stay_alive=True, ) @@ -269,3 +270,25 @@ def test_read_in_order(): node.query( "SELECT * FROM encrypted_test ORDER BY a, b SETTINGS optimize_read_in_order=0 FORMAT Null" ) + + +def test_restart(): + node.query( + """ + DROP TABLE IF EXISTS encrypted_test; + CREATE TABLE encrypted_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS disk='disk_s3_encrypted' + """ + ) + + node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") + select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.restart_clickhouse() + + assert node.query(select_query) == "(0,'data'),(1,'data')" From 1e462e02ed56652cd3f1fc636d2ae407aa150477 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 24 Mar 2023 17:00:04 +0100 Subject: [PATCH 27/73] add ut --- src/Common/tests/gtest_base_json.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/Common/tests/gtest_base_json.cpp diff --git a/src/Common/tests/gtest_base_json.cpp b/src/Common/tests/gtest_base_json.cpp new file mode 100644 index 00000000000..2a359bd43b4 --- /dev/null +++ b/src/Common/tests/gtest_base_json.cpp @@ -0,0 +1,14 @@ +#include + +#include + +TEST(JSON, searchField) +{ + const JSON json = JSON(std::string_view(R"({"k1":1,"k2":{"k3":2,"k4":3,"k":4},"k":5})")); + ASSERT_EQ(json["k1"], 1); + ASSERT_EQ(json["k2"].toString(), R"({"k3":2,"k4":3,"k":4})"); + ASSERT_EQ(json["k2"]["k3"].getUInt(), 2); + ASSERT_EQ(json["k2"]["k4"].getUInt(), 3); + ASSERT_EQ(json["k2"]["k"].getUInt(), 4); + ASSERT_EQ(json["k"].getUInt(), 5); +} From 5a4525742d86e8a2be642eaf7c141e95927ebdd3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Mar 2023 18:53:00 +0100 Subject: [PATCH 28/73] Drop table after test --- tests/integration/test_encrypted_disk/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 5b85888201b..8e2935a1603 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -292,3 +292,5 @@ def test_restart(): node.restart_clickhouse() assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY;") From d00587202dd2b1aad5188b09acdb4fe40c950cde Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 24 Mar 2023 19:25:07 +0000 Subject: [PATCH 29/73] Make better --- src/Interpreters/ProcessList.cpp | 17 +++++++---------- src/Interpreters/ProcessList.h | 3 +-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index d7ba0b8330f..a86377a37d5 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -388,12 +388,6 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } -bool QueryStatus::ExecutorHolder::equals(const PipelineExecutor * e) -{ - std::lock_guard lock(mutex); - return executor == e; -} - CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -434,19 +428,22 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }) == executors.end()); + assert(!executor_indexes.contains(e)); executors.push_back(std::make_shared(e)); + executor_indexes[e] = executors.size() - 1; } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { ExecutorHolderPtr executor_holder; + { std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }); - assert(it != executors.end()); - executor_holder = *it; + assert(executor_indexes.contains(e)); + executor_holder = executors[executor_indexes[e]]; + executor_indexes.erase(e); } + /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) /// We should do it with released executors_mutex to avoid possible lock order inversion. executor_holder->remove(); diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 98d7daa0949..9e5c91bf2ed 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -127,8 +127,6 @@ protected: void remove(); - bool equals(const PipelineExecutor * e); - PipelineExecutor * executor; std::mutex mutex; }; @@ -137,6 +135,7 @@ protected: /// Array of PipelineExecutors to be cancelled when a cancelQuery is received std::vector executors; + std::unordered_map executor_indexes; enum QueryStreamsStatus { From 6c3765c8b663554efde86816395385b97dcabb9c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 20:40:45 +0000 Subject: [PATCH 30/73] Small fix --- src/Coordination/KeeperDispatcher.cpp | 48 +++++++++++++++---------- src/Coordination/KeeperDispatcher.h | 16 ++------- src/Coordination/KeeperStateMachine.cpp | 4 ++- src/Coordination/KeeperStateMachine.h | 2 +- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index fc97bc5e0e1..69abf85ae4e 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -110,9 +110,9 @@ void KeeperDispatcher::requestThread() if (!coordination_settings->quorum_reads && request.request->isReadRequest()) { ++read_requests; - std::pair key{current_batch.back().session_id, current_batch.back().request->xid}; - std::lock_guard lock(read_mutex); - related_read_requests[key].push_back(request); + const auto & last_request = current_batch.back(); + std::lock_guard lock(read_request_queue_mutex); + read_request_queue[last_request.session_id][last_request.request->xid].push_back(request); } else current_batch.emplace_back(request); @@ -328,18 +328,24 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3, [this](const KeeperStorage::RequestForSession & request_for_session) { - std::lock_guard lock(read_mutex); - if (auto it = related_read_requests.find(std::pair{request_for_session.session_id, request_for_session.request->xid}); it != related_read_requests.end()) + /// check if we have queue of read requests depending on this request to be committed + std::lock_guard lock(read_request_queue_mutex); + if (auto it = read_request_queue.find(request_for_session.session_id); it != read_request_queue.end()) { - for (const auto & read_request : it->second) - { - if (server->isLeaderAlive()) - server->putLocalReadRequest(read_request); - else - addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); - } + auto & xid_to_request_queue = it->second; - related_read_requests.erase(it); + if (auto request_queue_it = xid_to_request_queue.find(request_for_session.request->xid); request_queue_it != xid_to_request_queue.end()) + { + for (const auto & read_request : request_queue_it->second) + { + if (server->isLeaderAlive()) + server->putLocalReadRequest(read_request); + else + addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); + } + + xid_to_request_queue.erase(request_queue_it); + } } }); @@ -554,12 +560,18 @@ void KeeperDispatcher::sessionCleanerTask() void KeeperDispatcher::finishSession(int64_t session_id) { - std::lock_guard lock(session_to_response_callback_mutex); - auto session_it = session_to_response_callback.find(session_id); - if (session_it != session_to_response_callback.end()) { - session_to_response_callback.erase(session_it); - CurrentMetrics::sub(CurrentMetrics::KeeperAliveConnections); + std::lock_guard lock(session_to_response_callback_mutex); + auto session_it = session_to_response_callback.find(session_id); + if (session_it != session_to_response_callback.end()) + { + session_to_response_callback.erase(session_it); + CurrentMetrics::sub(CurrentMetrics::KeeperAliveConnections); + } + } + { + std::lock_guard lock(read_request_queue_mutex); + read_request_queue.erase(session_id); } } diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index e7570727b9a..77b5510cbb3 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -104,20 +104,10 @@ private: void forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions); public: - std::mutex read_mutex; + std::mutex read_request_queue_mutex; - struct PairHash - { - auto operator()(std::pair pair) const - { - SipHash hash; - hash.update(pair.first); - hash.update(pair.second); - return hash.get64(); - } - }; - - std::unordered_map, KeeperStorage::RequestsForSessions, PairHash> related_read_requests; + /// queue of read requests that can be processed after a request with specific session ID and XID is committed + std::unordered_map> read_request_queue; /// Just allocate some objects, real initialization is done by `intialize method` KeeperDispatcher(); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 0b69b00bf0e..632aaec6b54 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -276,7 +276,9 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n ProfileEvents::increment(ProfileEvents::KeeperCommits); last_committed_idx = log_idx; - commit_callback(request_for_session); + + if (commit_callback) + commit_callback(request_for_session); return nullptr; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 6babf741dbd..e4f0295db99 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -31,7 +31,7 @@ public: const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, - CommitCallback commit_callback_, + CommitCallback commit_callback_ = {}, const std::string & superdigest_ = ""); /// Read state from the latest snapshot From 57ceb3436669ae60ac046eb562cd2627a5a98eb5 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Sat, 25 Mar 2023 13:36:58 +0000 Subject: [PATCH 31/73] Remove database from test --- ...6_ignore_inacc_tables_mat_view_atttach.sql | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index 48f4bf2b2e2..615737eacf0 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -1,34 +1,29 @@ SET send_logs_level = 'fatal'; -DROP DATABASE IF EXISTS test_db; +CREATE TABLE test_table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; -CREATE DATABASE test_db; +CREATE TABLE mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; -CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; +CREATE MATERIALIZED VIEW mview TO mview_backend AS SELECT n, n * n AS "n2" FROM test_table; -CREATE TABLE test_db.mview_backend (n Int32, n2 Int64) ENGINE MergeTree PARTITION BY n ORDER BY n; +DROP TABLE test_table; -CREATE MATERIALIZED VIEW test_db.mview TO test_db.mview_backend AS SELECT n, n * n AS "n2" FROM test_db.table; - -DROP TABLE test_db.table; - -DETACH TABLE test_db.mview; +DETACH TABLE mview; /* Check that we get an exception with the option. */ SET skip_materialized_view_checking_if_source_table_not_exist = 0; -ATTACH TABLE test_db.mview; --{serverError 60} +ATTACH TABLE mview; --{serverError 60} /* Check that we don't get an exception with the option. */ SET skip_materialized_view_checking_if_source_table_not_exist = 1; -ATTACH TABLE test_db.mview; +ATTACH TABLE mview; /* Check if the data in the materialized view is updated after the restore.*/ -CREATE TABLE test_db.table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; +CREATE TABLE test_table (n Int32, s String) ENGINE MergeTree PARTITION BY n ORDER BY n; -INSERT INTO test_db.table VALUES (3,'some_val'); +INSERT INTO test_table VALUES (3,'some_val'); -SELECT n,s FROM test_db.table ORDER BY n; -SELECT n,n2 FROM test_db.mview ORDER by n; +SELECT n,s FROM test_table ORDER BY n; +SELECT n,n2 FROM mview ORDER by n; -DROP DATABASE test_db; From 79e22d13fbaf495b83e10b4ca87363eb81f2635c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 27 Mar 2023 07:36:39 +0000 Subject: [PATCH 32/73] small fix --- src/Coordination/KeeperDispatcher.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 12248950c32..1828182751d 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -92,7 +92,7 @@ void KeeperDispatcher::requestThread() KeeperStorage::RequestsForSessions current_batch; size_t current_batch_bytes_size = 0; - bool has_read_request{false}; + bool has_read_request = false; /// If new request is not read request or we must to process it through quorum. /// Otherwise we will process it locally. @@ -101,8 +101,6 @@ void KeeperDispatcher::requestThread() current_batch_bytes_size += request.request->bytesSize(); current_batch.emplace_back(request); - size_t read_requests = 0; - const auto try_get_request = [&] { /// Trying to get batch requests as fast as possible @@ -112,7 +110,6 @@ void KeeperDispatcher::requestThread() /// Don't append read request into batch, we have to process them separately if (!coordination_settings->quorum_reads && request.request->isReadRequest()) { - ++read_requests; const auto & last_request = current_batch.back(); std::lock_guard lock(read_request_queue_mutex); read_request_queue[last_request.session_id][last_request.request->xid].push_back(request); From db55fa864acade1a27c6e1987c4135e85a08dffb Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 27 Mar 2023 10:47:34 +0200 Subject: [PATCH 33/73] fix build --- src/Common/tests/gtest_base_json.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/tests/gtest_base_json.cpp b/src/Common/tests/gtest_base_json.cpp index 2a359bd43b4..60c469f1542 100644 --- a/src/Common/tests/gtest_base_json.cpp +++ b/src/Common/tests/gtest_base_json.cpp @@ -5,7 +5,7 @@ TEST(JSON, searchField) { const JSON json = JSON(std::string_view(R"({"k1":1,"k2":{"k3":2,"k4":3,"k":4},"k":5})")); - ASSERT_EQ(json["k1"], 1); + ASSERT_EQ(json["k1"].getUInt(), 1); ASSERT_EQ(json["k2"].toString(), R"({"k3":2,"k4":3,"k":4})"); ASSERT_EQ(json["k2"]["k3"].getUInt(), 2); ASSERT_EQ(json["k2"]["k4"].getUInt(), 3); From 1963268fecad6fa100b4945172dff1604645d22e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 13:17:00 +0300 Subject: [PATCH 34/73] Update PULL_REQUEST_TEMPLATE.md There is no "prestable" version. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5d09d3a9ef3..449abc9484d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ tests/ci/run_check.py ### Changelog category (leave one): - New Feature - Improvement -- Bug Fix (user-visible misbehavior in official stable or prestable release) +- Bug Fix (user-visible misbehavior in an official stable release) - Performance Improvement - Backward Incompatible Change - Build/Testing/Packaging Improvement From e0252db8d457297ef64d849abd62be7b7efc8307 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 12:19:32 +0200 Subject: [PATCH 35/73] No prestable --- CHANGELOG.md | 2 +- docs/changelogs/v21.10.1.8013-prestable.md | 2 +- docs/changelogs/v21.10.2.15-stable.md | 2 +- docs/changelogs/v21.10.3.9-stable.md | 4 ++-- docs/changelogs/v21.10.4.26-stable.md | 2 +- docs/changelogs/v21.10.5.3-stable.md | 2 +- docs/changelogs/v21.10.6.2-stable.md | 4 ++-- docs/changelogs/v21.11.1.8636-prestable.md | 4 ++-- docs/changelogs/v21.11.11.1-stable.md | 2 +- docs/changelogs/v21.11.2.2-stable.md | 2 +- docs/changelogs/v21.11.3.6-stable.md | 2 +- docs/changelogs/v21.11.4.14-stable.md | 2 +- docs/changelogs/v21.11.5.33-stable.md | 2 +- docs/changelogs/v21.11.6.7-stable.md | 2 +- docs/changelogs/v21.11.7.9-stable.md | 4 ++-- docs/changelogs/v21.11.8.4-stable.md | 2 +- docs/changelogs/v21.11.9.1-stable.md | 2 +- docs/changelogs/v21.12.1.9017-prestable.md | 2 +- docs/changelogs/v21.12.2.17-stable.md | 4 ++-- docs/changelogs/v21.12.3.32-stable.md | 2 +- docs/changelogs/v21.12.4.1-stable.md | 2 +- docs/changelogs/v21.3.16.5-lts.md | 2 +- docs/changelogs/v21.3.17.2-lts.md | 2 +- docs/changelogs/v21.3.18.4-lts.md | 2 +- docs/changelogs/v21.3.19.1-lts.md | 4 ++-- docs/changelogs/v21.3.20.1-lts.md | 2 +- docs/changelogs/v21.6.9.7-stable.md | 2 +- docs/changelogs/v21.7.10.4-stable.md | 2 +- docs/changelogs/v21.7.11.3-stable.md | 2 +- docs/changelogs/v21.7.9.7-stable.md | 2 +- docs/changelogs/v21.8.10.19-lts.md | 2 +- docs/changelogs/v21.8.11.4-lts.md | 4 ++-- docs/changelogs/v21.8.12.29-lts.md | 2 +- docs/changelogs/v21.8.13.6-lts.md | 4 ++-- docs/changelogs/v21.8.14.5-lts.md | 2 +- docs/changelogs/v21.8.15.7-lts.md | 2 +- docs/changelogs/v21.8.5.7-lts.md | 2 +- docs/changelogs/v21.8.6.15-lts.md | 2 +- docs/changelogs/v21.8.7.22-lts.md | 2 +- docs/changelogs/v21.8.8.29-lts.md | 2 +- docs/changelogs/v21.8.9.13-lts.md | 2 +- docs/changelogs/v21.9.2.17-stable.md | 2 +- docs/changelogs/v21.9.3.30-stable.md | 2 +- docs/changelogs/v21.9.4.35-stable.md | 2 +- docs/changelogs/v21.9.5.16-stable.md | 2 +- docs/changelogs/v21.9.6.24-stable.md | 4 ++-- docs/changelogs/v22.1.1.2542-prestable.md | 4 ++-- docs/changelogs/v22.1.3.7-stable.md | 2 +- docs/changelogs/v22.1.4.30-stable.md | 2 +- docs/changelogs/v22.10.1.1877-stable.md | 4 ++-- docs/changelogs/v22.10.2.11-stable.md | 2 +- docs/changelogs/v22.10.3.27-stable.md | 2 +- docs/changelogs/v22.10.4.23-stable.md | 2 +- docs/changelogs/v22.10.5.54-stable.md | 3 +-- docs/changelogs/v22.10.6.3-stable.md | 3 +-- docs/changelogs/v22.10.7.13-stable.md | 3 +-- docs/changelogs/v22.11.1.1360-stable.md | 2 +- docs/changelogs/v22.11.2.30-stable.md | 2 +- docs/changelogs/v22.11.3.47-stable.md | 3 +-- docs/changelogs/v22.11.4.3-stable.md | 3 +-- docs/changelogs/v22.11.5.15-stable.md | 3 +-- docs/changelogs/v22.11.6.44-stable.md | 3 +-- docs/changelogs/v22.12.1.1752-stable.md | 2 +- docs/changelogs/v22.12.2.25-stable.md | 2 +- docs/changelogs/v22.12.3.5-stable.md | 3 +-- docs/changelogs/v22.12.4.76-stable.md | 3 +-- docs/changelogs/v22.12.5.34-stable.md | 3 +-- docs/changelogs/v22.2.1.2139-prestable.md | 2 +- docs/changelogs/v22.2.3.5-stable.md | 2 +- docs/changelogs/v22.3.1.1262-prestable.md | 2 +- docs/changelogs/v22.3.10.22-lts.md | 4 ++-- docs/changelogs/v22.3.11.12-lts.md | 2 +- docs/changelogs/v22.3.12.19-lts.md | 2 +- docs/changelogs/v22.3.13.80-lts.md | 4 ++-- docs/changelogs/v22.3.14.18-lts.md | 2 +- docs/changelogs/v22.3.14.23-lts.md | 2 +- docs/changelogs/v22.3.15.33-lts.md | 2 +- docs/changelogs/v22.3.16.1190-lts.md | 3 +-- docs/changelogs/v22.3.18.37-lts.md | 3 +-- docs/changelogs/v22.3.19.6-lts.md | 3 +-- docs/changelogs/v22.3.3.44-lts.md | 2 +- docs/changelogs/v22.3.4.20-lts.md | 2 +- docs/changelogs/v22.3.5.5-lts.md | 2 +- docs/changelogs/v22.3.6.5-lts.md | 2 +- docs/changelogs/v22.3.7.28-lts.md | 4 ++-- docs/changelogs/v22.3.8.39-lts.md | 4 ++-- docs/changelogs/v22.3.9.19-lts.md | 2 +- docs/changelogs/v22.4.1.2305-prestable.md | 2 +- docs/changelogs/v22.4.3.3-stable.md | 2 +- docs/changelogs/v22.4.4.7-stable.md | 2 +- docs/changelogs/v22.4.5.9-stable.md | 2 +- docs/changelogs/v22.4.6.53-stable.md | 4 ++-- docs/changelogs/v22.5.1.2079-stable.md | 4 ++-- docs/changelogs/v22.5.2.53-stable.md | 2 +- docs/changelogs/v22.5.3.21-stable.md | 2 +- docs/changelogs/v22.5.4.19-stable.md | 2 +- docs/changelogs/v22.6.1.1985-stable.md | 4 ++-- docs/changelogs/v22.6.2.12-stable.md | 2 +- docs/changelogs/v22.6.3.35-stable.md | 2 +- docs/changelogs/v22.6.4.35-stable.md | 2 +- docs/changelogs/v22.6.5.22-stable.md | 2 +- docs/changelogs/v22.6.6.16-stable.md | 2 +- docs/changelogs/v22.6.7.7-stable.md | 2 +- docs/changelogs/v22.6.8.35-stable.md | 2 +- docs/changelogs/v22.6.9.11-stable.md | 2 +- docs/changelogs/v22.7.1.2484-stable.md | 4 ++-- docs/changelogs/v22.7.2.15-stable.md | 2 +- docs/changelogs/v22.7.3.5-stable.md | 2 +- docs/changelogs/v22.7.4.16-stable.md | 2 +- docs/changelogs/v22.7.5.13-stable.md | 2 +- docs/changelogs/v22.7.6.74-stable.md | 2 +- docs/changelogs/v22.7.7.24-stable.md | 2 +- docs/changelogs/v22.8.1.2097-lts.md | 2 +- docs/changelogs/v22.8.10.29-lts.md | 2 +- docs/changelogs/v22.8.11.15-lts.md | 2 +- docs/changelogs/v22.8.12.45-lts.md | 3 +-- docs/changelogs/v22.8.13.20-lts.md | 3 +-- docs/changelogs/v22.8.14.53-lts.md | 3 +-- docs/changelogs/v22.8.15.23-lts.md | 3 +-- docs/changelogs/v22.8.3.13-lts.md | 2 +- docs/changelogs/v22.8.4.7-lts.md | 2 +- docs/changelogs/v22.8.5.29-lts.md | 2 +- docs/changelogs/v22.8.6.71-lts.md | 2 +- docs/changelogs/v22.8.7.34-lts.md | 2 +- docs/changelogs/v22.8.8.3-lts.md | 2 +- docs/changelogs/v22.8.9.24-lts.md | 2 +- docs/changelogs/v22.9.2.7-stable.md | 2 +- docs/changelogs/v22.9.3.18-stable.md | 2 +- docs/changelogs/v22.9.4.32-stable.md | 2 +- docs/changelogs/v22.9.5.25-stable.md | 2 +- docs/changelogs/v22.9.6.20-stable.md | 2 +- docs/changelogs/v22.9.7.34-stable.md | 2 +- docs/changelogs/v23.1.1.3077-stable.md | 3 +-- docs/changelogs/v23.1.2.9-stable.md | 3 +-- docs/changelogs/v23.1.3.5-stable.md | 3 +-- docs/changelogs/v23.1.4.58-stable.md | 3 +-- docs/changelogs/v23.1.5.24-stable.md | 3 +-- docs/changelogs/v23.2.1.2537-stable.md | 3 +-- docs/changelogs/v23.2.2.20-stable.md | 3 +-- docs/changelogs/v23.2.3.17-stable.md | 3 +-- docs/changelogs/v23.2.4.12-stable.md | 3 +-- tests/ci/run_check.py | 4 ++-- utils/changelog/changelog.py | 8 ++++++++ 143 files changed, 170 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e22377e2332..01bbafe2f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,7 +140,7 @@ * Upgrade Intel QPL from v0.3.0 to v1.0.0 2. Build libaccel-config and link it statically to QPL library instead of dynamically. [#45809](https://github.com/ClickHouse/ClickHouse/pull/45809) ([jasperzhu](https://github.com/jinjunzh)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Flush data exactly by `rabbitmq_flush_interval_ms` or by `rabbitmq_max_block_size` in `StorageRabbitMQ`. Closes [#42389](https://github.com/ClickHouse/ClickHouse/issues/42389). Closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#44404](https://github.com/ClickHouse/ClickHouse/pull/44404) ([Kseniia Sumarokova](https://github.com/kssenii)). * Use PODArray to render in sparkBar function, so we can control the memory usage. Close [#44467](https://github.com/ClickHouse/ClickHouse/issues/44467). [#44489](https://github.com/ClickHouse/ClickHouse/pull/44489) ([Duc Canh Le](https://github.com/canhld94)). diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md index 02ea593e02a..1c40aa67711 100644 --- a/docs/changelogs/v21.10.1.8013-prestable.md +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -108,7 +108,7 @@ sidebar_label: 2022 * Print out git status information at CMake configure stage. [#28047](https://github.com/ClickHouse/ClickHouse/pull/28047) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). * Add new log level `` for testing environments. [#28559](https://github.com/ClickHouse/ClickHouse/pull/28559) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md index 4ae5c8f5072..42402808260 100644 --- a/docs/changelogs/v21.10.2.15-stable.md +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#29970](https://github.com/ClickHouse/ClickHouse/issues/29970): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#30051](https://github.com/ClickHouse/ClickHouse/issues/30051): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29054](https://github.com/ClickHouse/ClickHouse/issues/29054): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28795](https://github.com/ClickHouse/ClickHouse/issues/28795): - Fix the number of arguments required by s2RectAdd and s2RectContains functions. [#28663](https://github.com/ClickHouse/ClickHouse/pull/28663) ([Bharat Nallan](https://github.com/bharatnc)). diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md index d0384d58e23..327e34ca64c 100644 --- a/docs/changelogs/v21.10.3.9-stable.md +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -18,11 +18,11 @@ sidebar_label: 2022 * Backported in [#30620](https://github.com/ClickHouse/ClickHouse/issues/30620): Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#31369](https://github.com/ClickHouse/ClickHouse/issues/31369): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30915](https://github.com/ClickHouse/ClickHouse/issues/30915): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30824](https://github.com/ClickHouse/ClickHouse/issues/30824): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30766](https://github.com/ClickHouse/ClickHouse/issues/30766): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md index 7d1cc93bb98..267f2109f6f 100644 --- a/docs/changelogs/v21.10.4.26-stable.md +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31573](https://github.com/ClickHouse/ClickHouse/issues/31573): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31518](https://github.com/ClickHouse/ClickHouse/issues/31518): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). * Backported in [#31554](https://github.com/ClickHouse/ClickHouse/issues/31554): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md index 88d3d70028e..7c717dfe838 100644 --- a/docs/changelogs/v21.10.5.3-stable.md +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32252](https://github.com/ClickHouse/ClickHouse/issues/32252): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#32346](https://github.com/ClickHouse/ClickHouse/issues/32346): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32151](https://github.com/ClickHouse/ClickHouse/issues/32151): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#32093](https://github.com/ClickHouse/ClickHouse/issues/32093): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md index 74f037b2f8f..0e8e934e2fa 100644 --- a/docs/changelogs/v21.10.6.2-stable.md +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32692](https://github.com/ClickHouse/ClickHouse/issues/32692): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32680](https://github.com/ClickHouse/ClickHouse/issues/32680): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32285](https://github.com/ClickHouse/ClickHouse/issues/32285): Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)). @@ -23,7 +23,7 @@ sidebar_label: 2022 * Backported in [#33182](https://github.com/ClickHouse/ClickHouse/issues/33182): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#33655](https://github.com/ClickHouse/ClickHouse/issues/33655): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md index 407a5196c1d..d6a435dd3ce 100644 --- a/docs/changelogs/v21.11.1.8636-prestable.md +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -124,11 +124,11 @@ sidebar_label: 2022 * Recursive submodules are no longer needed for ClickHouse. [#30315](https://github.com/ClickHouse/ClickHouse/pull/30315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Added docker image to build docs. [#30499](https://github.com/ClickHouse/ClickHouse/pull/30499) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). * Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md index 85a8975c6e7..76cd5239cba 100644 --- a/docs/changelogs/v21.11.11.1-stable.md +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33656](https://github.com/ClickHouse/ClickHouse/issues/33656): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md index bf02de235e4..44938addad5 100644 --- a/docs/changelogs/v21.11.2.2-stable.md +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31154](https://github.com/ClickHouse/ClickHouse/issues/31154): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). * Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md index e3886c4efac..f9cc64e2c2b 100644 --- a/docs/changelogs/v21.11.3.6-stable.md +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31246](https://github.com/ClickHouse/ClickHouse/issues/31246): Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31206](https://github.com/ClickHouse/ClickHouse/issues/31206): Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31202](https://github.com/ClickHouse/ClickHouse/issues/31202): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md index f05f43d9de1..8882832d1fe 100644 --- a/docs/changelogs/v21.11.4.14-stable.md +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31370](https://github.com/ClickHouse/ClickHouse/issues/31370): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31282](https://github.com/ClickHouse/ClickHouse/issues/31282): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#31237](https://github.com/ClickHouse/ClickHouse/issues/31237): Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md index 3780d5a2530..11e7f24dbb1 100644 --- a/docs/changelogs/v21.11.5.33-stable.md +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31572](https://github.com/ClickHouse/ClickHouse/issues/31572): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31517](https://github.com/ClickHouse/ClickHouse/issues/31517): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). * Backported in [#31551](https://github.com/ClickHouse/ClickHouse/issues/31551): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md index 1f1935d1865..cddd472076a 100644 --- a/docs/changelogs/v21.11.6.7-stable.md +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32254](https://github.com/ClickHouse/ClickHouse/issues/32254): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#32345](https://github.com/ClickHouse/ClickHouse/issues/32345): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32152](https://github.com/ClickHouse/ClickHouse/issues/32152): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#32298](https://github.com/ClickHouse/ClickHouse/issues/32298): Fix recursive user defined functions crash. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md index baa6b0290a5..596d16a28ef 100644 --- a/docs/changelogs/v21.11.7.9-stable.md +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -10,13 +10,13 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32691](https://github.com/ClickHouse/ClickHouse/issues/32691): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32711](https://github.com/ClickHouse/ClickHouse/issues/32711): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#32568](https://github.com/ClickHouse/ClickHouse/issues/32568): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)). * Backported in [#32732](https://github.com/ClickHouse/ClickHouse/issues/32732): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md index bd71374e870..28d413dd2c5 100644 --- a/docs/changelogs/v21.11.8.4-stable.md +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32679](https://github.com/ClickHouse/ClickHouse/issues/32679): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32543](https://github.com/ClickHouse/ClickHouse/issues/32543): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md index 1473a4a152c..ea36479c943 100644 --- a/docs/changelogs/v21.11.9.1-stable.md +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md index e8f2ca283a4..88b8260e312 100644 --- a/docs/changelogs/v21.12.1.9017-prestable.md +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -132,7 +132,7 @@ sidebar_label: 2022 * Build rpm and tgz packages in master and release branches workfolw. [#32048](https://github.com/ClickHouse/ClickHouse/pull/32048) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix broken symlink for sysroot/linux-riscv64/usr/lib. [#32071](https://github.com/ClickHouse/ClickHouse/pull/32071) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). * Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md index 94cfc1b88a8..67761ce0e08 100644 --- a/docs/changelogs/v21.12.2.17-stable.md +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32693](https://github.com/ClickHouse/ClickHouse/issues/32693): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32681](https://github.com/ClickHouse/ClickHouse/issues/32681): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32483](https://github.com/ClickHouse/ClickHouse/issues/32483): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). @@ -23,7 +23,7 @@ sidebar_label: 2022 * Backported in [#32733](https://github.com/ClickHouse/ClickHouse/issues/32733): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#32793](https://github.com/ClickHouse/ClickHouse/issues/32793): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md index ea11efa46c5..c8c423a77b9 100644 --- a/docs/changelogs/v21.12.3.32-stable.md +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - ClickHouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32890](https://github.com/ClickHouse/ClickHouse/issues/32890): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#33183](https://github.com/ClickHouse/ClickHouse/issues/33183): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md index bd38dbd0c59..3345f76b317 100644 --- a/docs/changelogs/v21.12.4.1-stable.md +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#33551](https://github.com/ClickHouse/ClickHouse/issues/33551): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33537](https://github.com/ClickHouse/ClickHouse/issues/33537): Fix ORC stripe reading. [#32929](https://github.com/ClickHouse/ClickHouse/pull/32929) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). * Backported in [#33654](https://github.com/ClickHouse/ClickHouse/issues/33654): Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md index 123b27097d4..6aedeff5acb 100644 --- a/docs/changelogs/v21.3.16.5-lts.md +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28075](https://github.com/ClickHouse/ClickHouse/issues/28075): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). * Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md index 6c288b5a0d8..9104ae7aa0a 100644 --- a/docs/changelogs/v21.3.17.2-lts.md +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#28647](https://github.com/ClickHouse/ClickHouse/issues/28647): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28569](https://github.com/ClickHouse/ClickHouse/issues/28569): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). * Backported in [#28857](https://github.com/ClickHouse/ClickHouse/issues/28857): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md index d14dddfb1a5..33f4b86d81c 100644 --- a/docs/changelogs/v21.3.18.4-lts.md +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#30041](https://github.com/ClickHouse/ClickHouse/issues/30041): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29260](https://github.com/ClickHouse/ClickHouse/issues/29260): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29026](https://github.com/ClickHouse/ClickHouse/issues/29026): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md index 4d4404077a5..26c36725610 100644 --- a/docs/changelogs/v21.3.19.1-lts.md +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -14,11 +14,11 @@ sidebar_label: 2022 * Backported in [#31577](https://github.com/ClickHouse/ClickHouse/issues/31577): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). * Backported in [#32347](https://github.com/ClickHouse/ClickHouse/issues/32347): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30913](https://github.com/ClickHouse/ClickHouse/issues/30913): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30750](https://github.com/ClickHouse/ClickHouse/issues/30750): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#31038](https://github.com/ClickHouse/ClickHouse/issues/31038): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md index f9ce3cba78b..1b235556faf 100644 --- a/docs/changelogs/v21.3.20.1-lts.md +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32690](https://github.com/ClickHouse/ClickHouse/issues/32690): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). * Backported in [#33727](https://github.com/ClickHouse/ClickHouse/issues/33727): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md index ca1edeb1722..0a989e4d6b7 100644 --- a/docs/changelogs/v21.6.9.7-stable.md +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -40,7 +40,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28030](https://github.com/ClickHouse/ClickHouse/issues/28030): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28119](https://github.com/ClickHouse/ClickHouse/issues/28119): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#28179](https://github.com/ClickHouse/ClickHouse/issues/28179): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md index daa063ebf5a..238dfd651eb 100644 --- a/docs/changelogs/v21.7.10.4-stable.md +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#27925](https://github.com/ClickHouse/ClickHouse/issues/27925): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28752](https://github.com/ClickHouse/ClickHouse/issues/28752): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28509](https://github.com/ClickHouse/ClickHouse/issues/28509): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#28570](https://github.com/ClickHouse/ClickHouse/issues/28570): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md index b3d1c9a44fd..8ccc31657de 100644 --- a/docs/changelogs/v21.7.11.3-stable.md +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md index ac985f7af37..7aaab54af6b 100644 --- a/docs/changelogs/v21.7.9.7-stable.md +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -23,7 +23,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28032](https://github.com/ClickHouse/ClickHouse/issues/28032): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28116](https://github.com/ClickHouse/ClickHouse/issues/28116): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#28183](https://github.com/ClickHouse/ClickHouse/issues/28183): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md index 56c682ddbd2..5873f2b2ff9 100644 --- a/docs/changelogs/v21.8.10.19-lts.md +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#30452](https://github.com/ClickHouse/ClickHouse/issues/30452): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29724](https://github.com/ClickHouse/ClickHouse/issues/29724): Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30233](https://github.com/ClickHouse/ClickHouse/issues/30233): Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md index d88d191bae2..81bbea4ee21 100644 --- a/docs/changelogs/v21.8.11.4-lts.md +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -16,11 +16,11 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31368](https://github.com/ClickHouse/ClickHouse/issues/31368): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30914](https://github.com/ClickHouse/ClickHouse/issues/30914): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28756](https://github.com/ClickHouse/ClickHouse/issues/28756): Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)). * Backported in [#30825](https://github.com/ClickHouse/ClickHouse/issues/30825): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md index bd1f0c7fe60..8b68a6a3af8 100644 --- a/docs/changelogs/v21.8.12.29-lts.md +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31575](https://github.com/ClickHouse/ClickHouse/issues/31575): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31204](https://github.com/ClickHouse/ClickHouse/issues/31204): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31253](https://github.com/ClickHouse/ClickHouse/issues/31253): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md index 63ac956c3d5..205628c6330 100644 --- a/docs/changelogs/v21.8.13.6-lts.md +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32688](https://github.com/ClickHouse/ClickHouse/issues/32688): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). * Backported in [#32343](https://github.com/ClickHouse/ClickHouse/issues/32343): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32108](https://github.com/ClickHouse/ClickHouse/issues/32108): Fix crash with empty result on odbc query. Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#32150](https://github.com/ClickHouse/ClickHouse/issues/32150): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). @@ -26,7 +26,7 @@ sidebar_label: 2022 * Backported in [#33048](https://github.com/ClickHouse/ClickHouse/issues/33048): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#32795](https://github.com/ClickHouse/ClickHouse/issues/32795): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32659](https://github.com/ClickHouse/ClickHouse/issues/32659): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md index 1012d9c5784..75d966ec9e7 100644 --- a/docs/changelogs/v21.8.14.5-lts.md +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33184](https://github.com/ClickHouse/ClickHouse/issues/33184): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#33659](https://github.com/ClickHouse/ClickHouse/issues/33659): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md index eb6bf39d7be..1522c28016a 100644 --- a/docs/changelogs/v21.8.15.7-lts.md +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34121](https://github.com/ClickHouse/ClickHouse/issues/34121): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#34097](https://github.com/ClickHouse/ClickHouse/issues/34097): Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md index 00c6c6e46a7..fa459e093f7 100644 --- a/docs/changelogs/v21.8.5.7-lts.md +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28031](https://github.com/ClickHouse/ClickHouse/issues/28031): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#27974](https://github.com/ClickHouse/ClickHouse/issues/27974): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28117](https://github.com/ClickHouse/ClickHouse/issues/28117): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md index dc8ae4ec9b5..aa51dc1ac1e 100644 --- a/docs/changelogs/v21.8.6.15-lts.md +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2022 * Backported in [#27923](https://github.com/ClickHouse/ClickHouse/issues/27923): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28753](https://github.com/ClickHouse/ClickHouse/issues/28753): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28644](https://github.com/ClickHouse/ClickHouse/issues/28644): Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)). * Backported in [#28508](https://github.com/ClickHouse/ClickHouse/issues/28508): Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md index b6c5b70e096..7a751be4132 100644 --- a/docs/changelogs/v21.8.7.22-lts.md +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29121](https://github.com/ClickHouse/ClickHouse/issues/29121): Better check for connection usability and also catch any exception in RabbitMQ shutdown just in case. [#28797](https://github.com/ClickHouse/ClickHouse/pull/28797) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md index 160d30a6aa9..e988c3c6801 100644 --- a/docs/changelogs/v21.8.8.29-lts.md +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#29128](https://github.com/ClickHouse/ClickHouse/issues/29128): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29262](https://github.com/ClickHouse/ClickHouse/issues/29262): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29106](https://github.com/ClickHouse/ClickHouse/issues/29106): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md index a48ca30080e..71919c48c47 100644 --- a/docs/changelogs/v21.8.9.13-lts.md +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#29817](https://github.com/ClickHouse/ClickHouse/issues/29817): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#29973](https://github.com/ClickHouse/ClickHouse/issues/29973): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29676](https://github.com/ClickHouse/ClickHouse/issues/29676): Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29538](https://github.com/ClickHouse/ClickHouse/issues/29538): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md index 39e3f627f4a..08d208ec97b 100644 --- a/docs/changelogs/v21.9.2.17-stable.md +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28029](https://github.com/ClickHouse/ClickHouse/issues/28029): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#27973](https://github.com/ClickHouse/ClickHouse/issues/27973): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28118](https://github.com/ClickHouse/ClickHouse/issues/28118): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md index ee2dd24277d..28375c5588e 100644 --- a/docs/changelogs/v21.9.3.30-stable.md +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#28897](https://github.com/ClickHouse/ClickHouse/issues/28897): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28815](https://github.com/ClickHouse/ClickHouse/issues/28815): Fix possible crash for `SELECT` with partially created aggregate projection in case of exception. [#28700](https://github.com/ClickHouse/ClickHouse/pull/28700) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28789](https://github.com/ClickHouse/ClickHouse/issues/28789): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md index 5a556df1b3a..0b300574559 100644 --- a/docs/changelogs/v21.9.4.35-stable.md +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md index e681322eb45..895e882d257 100644 --- a/docs/changelogs/v21.9.5.16-stable.md +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#29972](https://github.com/ClickHouse/ClickHouse/issues/29972): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#30052](https://github.com/ClickHouse/ClickHouse/issues/30052): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29055](https://github.com/ClickHouse/ClickHouse/issues/29055): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29107](https://github.com/ClickHouse/ClickHouse/issues/29107): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md index 2407c031873..890dc8d5d02 100644 --- a/docs/changelogs/v21.9.6.24-stable.md +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -21,11 +21,11 @@ sidebar_label: 2022 * Backported in [#31371](https://github.com/ClickHouse/ClickHouse/issues/31371): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#31576](https://github.com/ClickHouse/ClickHouse/issues/31576): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30916](https://github.com/ClickHouse/ClickHouse/issues/30916): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30823](https://github.com/ClickHouse/ClickHouse/issues/30823): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30609](https://github.com/ClickHouse/ClickHouse/issues/30609): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index 8d7bb015db6..cacd13c1e12 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -118,7 +118,7 @@ sidebar_label: 2022 * Remove editing /etc/hosts from Dockerfile. [#33635](https://github.com/ClickHouse/ClickHouse/pull/33635) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Properly separate thrift-cmake from arrow-cmake after https://github.com/ClickHouse/ClickHouse/pull/31104 . cc @taiyang-li. [#33661](https://github.com/ClickHouse/ClickHouse/pull/33661) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fixed CAST from String to IPv4 or IPv6 and back. Fixed error message in case of failed conversion. [#29224](https://github.com/ClickHouse/ClickHouse/pull/29224) ([Dmitry Novik](https://github.com/novikd)). * Fix base64Encode adding trailing bytes on small strings. [#31797](https://github.com/ClickHouse/ClickHouse/pull/31797) ([Kevin Michel](https://github.com/kmichel-aiven)). @@ -185,7 +185,7 @@ sidebar_label: 2022 * Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). * Fix wrong database for JOIN w/o explicit database in distributed queries (Fixes: [#10471](https://github.com/ClickHouse/ClickHouse/issues/10471)). [#33611](https://github.com/ClickHouse/ClickHouse/pull/33611) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index a92a82be290..fd8787f0e75 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -10,6 +10,6 @@ sidebar_label: 2022 #### Improvement * Backported in [#33793](https://github.com/ClickHouse/ClickHouse/issues/33793): Create parent directories in DiskS3::restoreFileOperations method. [#33730](https://github.com/ClickHouse/ClickHouse/pull/33730) ([ianton-ru](https://github.com/ianton-ru)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33898](https://github.com/ClickHouse/ClickHouse/issues/33898): Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.1.4.30-stable.md b/docs/changelogs/v22.1.4.30-stable.md index c4286d7b64d..b0437382a46 100644 --- a/docs/changelogs/v22.1.4.30-stable.md +++ b/docs/changelogs/v22.1.4.30-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backport CI checks to 22.1 release branch. [#34897](https://github.com/ClickHouse/ClickHouse/pull/34897) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34119](https://github.com/ClickHouse/ClickHouse/issues/34119): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#34124](https://github.com/ClickHouse/ClickHouse/issues/34124): Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.10.1.1877-stable.md b/docs/changelogs/v22.10.1.1877-stable.md index 5b573a3faa4..23c58447c99 100644 --- a/docs/changelogs/v22.10.1.1877-stable.md +++ b/docs/changelogs/v22.10.1.1877-stable.md @@ -105,7 +105,7 @@ sidebar_label: 2022 * Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix power8 support. [#42462](https://github.com/ClickHouse/ClickHouse/pull/42462) ([Boris Kuschel](https://github.com/bkuschel)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Several fixes for DiskWeb. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -147,7 +147,7 @@ sidebar_label: 2022 * This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix function `arrayElement` with type `Map` with `Nullable` values and `Nullable` index. [#42623](https://github.com/ClickHouse/ClickHouse/pull/42623) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v22.10.2.11-stable.md b/docs/changelogs/v22.10.2.11-stable.md index 4e3c382f5a7..196d3fbde80 100644 --- a/docs/changelogs/v22.10.2.11-stable.md +++ b/docs/changelogs/v22.10.2.11-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.10.2.11-stable (d2bfcaba002) FIXME as compared to v22.10.1.1877-stable (98ab5a3c189) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42750](https://github.com/ClickHouse/ClickHouse/issues/42750): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#42793](https://github.com/ClickHouse/ClickHouse/issues/42793): Fix a bug in ParserFunction that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)). diff --git a/docs/changelogs/v22.10.3.27-stable.md b/docs/changelogs/v22.10.3.27-stable.md index 6dc9fd7f3b9..6e0188ad619 100644 --- a/docs/changelogs/v22.10.3.27-stable.md +++ b/docs/changelogs/v22.10.3.27-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42959](https://github.com/ClickHouse/ClickHouse/issues/42959): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43042](https://github.com/ClickHouse/ClickHouse/issues/43042): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42864](https://github.com/ClickHouse/ClickHouse/issues/42864): Fix lowerUTF8()/upperUTF8() in case of symbol was in between 16-byte boundary (very frequent case of you have strings > 16 bytes long). [#42812](https://github.com/ClickHouse/ClickHouse/pull/42812) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#43173](https://github.com/ClickHouse/ClickHouse/issues/43173): Fix rare possible hung on query cancellation. [#42874](https://github.com/ClickHouse/ClickHouse/pull/42874) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.10.4.23-stable.md b/docs/changelogs/v22.10.4.23-stable.md index 04eb8be982f..4438a3470fc 100644 --- a/docs/changelogs/v22.10.4.23-stable.md +++ b/docs/changelogs/v22.10.4.23-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43053](https://github.com/ClickHouse/ClickHouse/issues/43053): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43715](https://github.com/ClickHouse/ClickHouse/issues/43715): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43576](https://github.com/ClickHouse/ClickHouse/issues/43576): Fix possible `Cannot create non-empty column with type Nothing` in functions if/multiIf. Closes [#43356](https://github.com/ClickHouse/ClickHouse/issues/43356). [#43368](https://github.com/ClickHouse/ClickHouse/pull/43368) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.10.5.54-stable.md b/docs/changelogs/v22.10.5.54-stable.md index e372fb30618..73c15cf985d 100644 --- a/docs/changelogs/v22.10.5.54-stable.md +++ b/docs/changelogs/v22.10.5.54-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#44379](https://github.com/ClickHouse/ClickHouse/issues/44379): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44559](https://github.com/ClickHouse/ClickHouse/issues/44559): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44754](https://github.com/ClickHouse/ClickHouse/issues/44754): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43527](https://github.com/ClickHouse/ClickHouse/issues/43527): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -41,4 +41,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.10.6.3-stable.md b/docs/changelogs/v22.10.6.3-stable.md index b0e88c92cb0..c09e9a8670b 100644 --- a/docs/changelogs/v22.10.6.3-stable.md +++ b/docs/changelogs/v22.10.6.3-stable.md @@ -7,7 +7,6 @@ sidebar_label: 2023 ### ClickHouse release v22.10.6.3-stable (645a66d221f) FIXME as compared to v22.10.5.54-stable (dbc7984dc3b) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45084](https://github.com/ClickHouse/ClickHouse/issues/45084): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.10.7.13-stable.md b/docs/changelogs/v22.10.7.13-stable.md index c906e00e524..987bf6cf1e1 100644 --- a/docs/changelogs/v22.10.7.13-stable.md +++ b/docs/changelogs/v22.10.7.13-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.10.7.13-stable (d261d9036cc) FIXME as compared to v22.10.6.3-stable (645a66d221f) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44998](https://github.com/ClickHouse/ClickHouse/issues/44998): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45551](https://github.com/ClickHouse/ClickHouse/issues/45551): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -18,4 +18,3 @@ sidebar_label: 2023 * Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.1.1360-stable.md b/docs/changelogs/v22.11.1.1360-stable.md index 1da53be02b7..4aa110484f8 100644 --- a/docs/changelogs/v22.11.1.1360-stable.md +++ b/docs/changelogs/v22.11.1.1360-stable.md @@ -81,7 +81,7 @@ sidebar_label: 2022 * Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix schema inference in s3Cluster and improve in hdfsCluster. [#41979](https://github.com/ClickHouse/ClickHouse/pull/41979) ([Kruglov Pavel](https://github.com/Avogar)). * Fix retries while reading from http table engines / table function. (retrtiable errors could be retries more times than needed, non-retrialble errors resulted in failed assertion in code). [#42224](https://github.com/ClickHouse/ClickHouse/pull/42224) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.11.2.30-stable.md b/docs/changelogs/v22.11.2.30-stable.md index 7b2febe072a..4759aa4a503 100644 --- a/docs/changelogs/v22.11.2.30-stable.md +++ b/docs/changelogs/v22.11.2.30-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#43511](https://github.com/ClickHouse/ClickHouse/issues/43511): Restrict default access to named collections for user defined in config. It must have explicit `show_named_collections=1` to be able to see them. [#43325](https://github.com/ClickHouse/ClickHouse/pull/43325) ([Kseniia Sumarokova](https://github.com/kssenii)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43716](https://github.com/ClickHouse/ClickHouse/issues/43716): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43431](https://github.com/ClickHouse/ClickHouse/issues/43431): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.11.3.47-stable.md b/docs/changelogs/v22.11.3.47-stable.md index d6451b853f7..a993ff8516f 100644 --- a/docs/changelogs/v22.11.3.47-stable.md +++ b/docs/changelogs/v22.11.3.47-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2023 * Backported in [#44380](https://github.com/ClickHouse/ClickHouse/issues/44380): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44560](https://github.com/ClickHouse/ClickHouse/issues/44560): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44756](https://github.com/ClickHouse/ClickHouse/issues/44756): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43528](https://github.com/ClickHouse/ClickHouse/issues/43528): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -37,4 +37,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.11.4.3-stable.md b/docs/changelogs/v22.11.4.3-stable.md index 33780e848ef..b0e7586277a 100644 --- a/docs/changelogs/v22.11.4.3-stable.md +++ b/docs/changelogs/v22.11.4.3-stable.md @@ -7,7 +7,6 @@ sidebar_label: 2023 ### ClickHouse release v22.11.4.3-stable (7f4cf554f69) FIXME as compared to v22.11.3.47-stable (1c49d124a37) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45085](https://github.com/ClickHouse/ClickHouse/issues/45085): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.11.5.15-stable.md b/docs/changelogs/v22.11.5.15-stable.md index 742a8740514..f11a01c4b1c 100644 --- a/docs/changelogs/v22.11.5.15-stable.md +++ b/docs/changelogs/v22.11.5.15-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.11.5.15-stable (d763e5a9239) FIXME as compared to v22.11.4.3-stable (7f4cf554f69) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44999](https://github.com/ClickHouse/ClickHouse/issues/44999): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45552](https://github.com/ClickHouse/ClickHouse/issues/45552): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -19,4 +19,3 @@ sidebar_label: 2023 * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.6.44-stable.md b/docs/changelogs/v22.11.6.44-stable.md index 6e628b85150..db19e73c666 100644 --- a/docs/changelogs/v22.11.6.44-stable.md +++ b/docs/changelogs/v22.11.6.44-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#46483](https://github.com/ClickHouse/ClickHouse/issues/46483): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46507](https://github.com/ClickHouse/ClickHouse/issues/46507): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45903](https://github.com/ClickHouse/ClickHouse/issues/45903): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#46239](https://github.com/ClickHouse/ClickHouse/issues/46239): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). @@ -34,4 +34,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.1.1752-stable.md b/docs/changelogs/v22.12.1.1752-stable.md index 4f4c4b11150..1549af037f2 100644 --- a/docs/changelogs/v22.12.1.1752-stable.md +++ b/docs/changelogs/v22.12.1.1752-stable.md @@ -91,7 +91,7 @@ sidebar_label: 2022 * Bring sha512 sums back to the building step. [#44017](https://github.com/ClickHouse/ClickHouse/pull/44017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Kill stress tests after 2.5h in case of hanging process. [#44214](https://github.com/ClickHouse/ClickHouse/pull/44214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fixed unable to log in (because of failure to create session_log entry) in rare case of messed up setting profiles. ... [#42641](https://github.com/ClickHouse/ClickHouse/pull/42641) ([Vasily Nemkov](https://github.com/Enmk)). * Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.12.2.25-stable.md b/docs/changelogs/v22.12.2.25-stable.md index 194d0fe3cbc..968854d0428 100644 --- a/docs/changelogs/v22.12.2.25-stable.md +++ b/docs/changelogs/v22.12.2.25-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2023 * Backported in [#44381](https://github.com/ClickHouse/ClickHouse/issues/44381): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44561](https://github.com/ClickHouse/ClickHouse/issues/44561): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44739](https://github.com/ClickHouse/ClickHouse/issues/44739): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#44764](https://github.com/ClickHouse/ClickHouse/issues/44764): Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.12.3.5-stable.md b/docs/changelogs/v22.12.3.5-stable.md index 8cbcbc6a590..5f0cc9cebf3 100644 --- a/docs/changelogs/v22.12.3.5-stable.md +++ b/docs/changelogs/v22.12.3.5-stable.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v22.12.3.5-stable (893de538f02) FIXME as compared to v22.12.2.25-stable (c790cfd4465) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45086](https://github.com/ClickHouse/ClickHouse/issues/45086): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.12.4.76-stable.md b/docs/changelogs/v22.12.4.76-stable.md index 79569ff841e..cdadaae7f7b 100644 --- a/docs/changelogs/v22.12.4.76-stable.md +++ b/docs/changelogs/v22.12.4.76-stable.md @@ -22,7 +22,7 @@ sidebar_label: 2023 * Backported in [#46509](https://github.com/ClickHouse/ClickHouse/issues/46509): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#47058](https://github.com/ClickHouse/ClickHouse/issues/47058): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45904](https://github.com/ClickHouse/ClickHouse/issues/45904): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#45321](https://github.com/ClickHouse/ClickHouse/issues/45321): Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)). @@ -52,4 +52,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.5.34-stable.md b/docs/changelogs/v22.12.5.34-stable.md index 95befaa88ff..61f099462a0 100644 --- a/docs/changelogs/v22.12.5.34-stable.md +++ b/docs/changelogs/v22.12.5.34-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2023 #### Improvement * Backported in [#46983](https://github.com/ClickHouse/ClickHouse/issues/46983): - Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45729](https://github.com/ClickHouse/ClickHouse/issues/45729): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). * Backported in [#46398](https://github.com/ClickHouse/ClickHouse/issues/46398): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -26,4 +26,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index 67db2d9a18d..ca3a84ceaa2 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -141,7 +141,7 @@ sidebar_label: 2022 * - Rework version_helper, make it executable - Reimplement StorageSystemContributors.sh in version_helper - Create a release script. [#34641](https://github.com/ClickHouse/ClickHouse/pull/34641) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * - Fix path in workflows/release.yml - To be backported to branch 22.1. [#34646](https://github.com/ClickHouse/ClickHouse/pull/34646) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix lz4 compression for output. Closes [#31421](https://github.com/ClickHouse/ClickHouse/issues/31421). [#31862](https://github.com/ClickHouse/ClickHouse/pull/31862) ([Kruglov Pavel](https://github.com/Avogar)). * Create a function escapeForLDAPFilter and use it to escape characters '(' and ')' in a final_user_dn variable. [#33401](https://github.com/ClickHouse/ClickHouse/pull/33401) ([IlyaTsoi](https://github.com/IlyaTsoi)). diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index c433669049d..a4368e465aa 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.2.3.5-stable FIXME as compared to v22.2.2.1-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34848](https://github.com/ClickHouse/ClickHouse/issues/34848): Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index e6203ff18d1..385393cef17 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -95,7 +95,7 @@ sidebar_label: 2022 * Clion has the following problems "The breakpoint will not currently be hit. No executable code is associated with this line". [#35179](https://github.com/ClickHouse/ClickHouse/pull/35179) ([小路](https://github.com/nicelulu)). * Add an ability to build stripped binaries with cmake. [#35196](https://github.com/ClickHouse/ClickHouse/pull/35196) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix distributed subquery max_query_size limitation inconsistency. [#34078](https://github.com/ClickHouse/ClickHouse/pull/34078) ([Chao Ma](https://github.com/godliness)). * Fix incorrect trivial count result when part movement feature is used [#34089](https://github.com/ClickHouse/ClickHouse/issues/34089). [#34385](https://github.com/ClickHouse/ClickHouse/pull/34385) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v22.3.10.22-lts.md b/docs/changelogs/v22.3.10.22-lts.md index a43b8301aad..cc033eb707d 100644 --- a/docs/changelogs/v22.3.10.22-lts.md +++ b/docs/changelogs/v22.3.10.22-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#39761](https://github.com/ClickHouse/ClickHouse/issues/39761): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39206](https://github.com/ClickHouse/ClickHouse/issues/39206): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#39381](https://github.com/ClickHouse/ClickHouse/issues/39381): Fixed error `Not found column Type in block` in selects with `PREWHERE` and read-in-order optimizations. [#39157](https://github.com/ClickHouse/ClickHouse/pull/39157) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#39610](https://github.com/ClickHouse/ClickHouse/issues/39610): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)). * Backported in [#39834](https://github.com/ClickHouse/ClickHouse/issues/39834): Fix `CANNOT_READ_ALL_DATA` exception with `local_filesystem_read_method=pread_threadpool`. This bug affected only Linux kernel version 5.9 and 5.10 according to [man](https://manpages.debian.org/testing/manpages-dev/preadv2.2.en.html#BUGS). [#39800](https://github.com/ClickHouse/ClickHouse/pull/39800) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#39238](https://github.com/ClickHouse/ClickHouse/issues/39238): Fix performance regression of scalar query optimization. [#35986](https://github.com/ClickHouse/ClickHouse/pull/35986) ([Amos Bird](https://github.com/amosbird)). * Backported in [#39531](https://github.com/ClickHouse/ClickHouse/issues/39531): Fix some issues with async reads from remote filesystem which happened when reading low cardinality. [#36763](https://github.com/ClickHouse/ClickHouse/pull/36763) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.3.11.12-lts.md b/docs/changelogs/v22.3.11.12-lts.md index e718493d9a0..58df0c0cadb 100644 --- a/docs/changelogs/v22.3.11.12-lts.md +++ b/docs/changelogs/v22.3.11.12-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39881](https://github.com/ClickHouse/ClickHouse/issues/39881): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39336](https://github.com/ClickHouse/ClickHouse/issues/39336): Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.3.12.19-lts.md b/docs/changelogs/v22.3.12.19-lts.md index 4f6342419f7..6ae342583a9 100644 --- a/docs/changelogs/v22.3.12.19-lts.md +++ b/docs/changelogs/v22.3.12.19-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40695](https://github.com/ClickHouse/ClickHouse/issues/40695): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40160](https://github.com/ClickHouse/ClickHouse/issues/40160): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40122](https://github.com/ClickHouse/ClickHouse/issues/40122): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.3.13.80-lts.md b/docs/changelogs/v22.3.13.80-lts.md index bb84da9a96a..3d0a6a77276 100644 --- a/docs/changelogs/v22.3.13.80-lts.md +++ b/docs/changelogs/v22.3.13.80-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41557](https://github.com/ClickHouse/ClickHouse/issues/41557): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40745](https://github.com/ClickHouse/ClickHouse/issues/40745): * Fix cast lowcard of nullable in JoinSwitcher, close [#37385](https://github.com/ClickHouse/ClickHouse/issues/37385). [#37453](https://github.com/ClickHouse/ClickHouse/pull/37453) ([Vladimir C](https://github.com/vdimir)). * Backported in [#41812](https://github.com/ClickHouse/ClickHouse/issues/41812): Update `simdjson`. This fixes [#38621](https://github.com/ClickHouse/ClickHouse/issues/38621). [#38838](https://github.com/ClickHouse/ClickHouse/pull/38838) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -36,7 +36,7 @@ sidebar_label: 2022 * Backported in [#41639](https://github.com/ClickHouse/ClickHouse/issues/41639): Fix possible `pipeline stuck` exception for queries with `OFFSET`. The error was found with `enable_optimize_predicate_expression = 0` and always false condition in `WHERE`. Fixes [#41383](https://github.com/ClickHouse/ClickHouse/issues/41383). [#41588](https://github.com/ClickHouse/ClickHouse/pull/41588) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#41899](https://github.com/ClickHouse/ClickHouse/issues/41899): Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#41321](https://github.com/ClickHouse/ClickHouse/issues/41321): Fix bug in function `if` when resulting column type differs with resulting data type that led to logical errors like `Logical error: 'Bad cast from type DB::ColumnVector to DB::ColumnVector'.`. Closes [#35367](https://github.com/ClickHouse/ClickHouse/issues/35367). [#35476](https://github.com/ClickHouse/ClickHouse/pull/35476) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.3.14.18-lts.md b/docs/changelogs/v22.3.14.18-lts.md index 88801b268ce..235525bbe81 100644 --- a/docs/changelogs/v22.3.14.18-lts.md +++ b/docs/changelogs/v22.3.14.18-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.3.14.23-lts.md b/docs/changelogs/v22.3.14.23-lts.md index 0a8c645702e..e086e223fb1 100644 --- a/docs/changelogs/v22.3.14.23-lts.md +++ b/docs/changelogs/v22.3.14.23-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.3.15.33-lts.md b/docs/changelogs/v22.3.15.33-lts.md index e59bf8bd1cd..3f675bfba1e 100644 --- a/docs/changelogs/v22.3.15.33-lts.md +++ b/docs/changelogs/v22.3.15.33-lts.md @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#42963](https://github.com/ClickHouse/ClickHouse/issues/42963): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43039](https://github.com/ClickHouse/ClickHouse/issues/43039): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43427](https://github.com/ClickHouse/ClickHouse/issues/43427): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). * Backported in [#43720](https://github.com/ClickHouse/ClickHouse/issues/43720): Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). diff --git a/docs/changelogs/v22.3.16.1190-lts.md b/docs/changelogs/v22.3.16.1190-lts.md index a43d34551ca..b65a6484109 100644 --- a/docs/changelogs/v22.3.16.1190-lts.md +++ b/docs/changelogs/v22.3.16.1190-lts.md @@ -18,7 +18,7 @@ sidebar_label: 2023 * Backported in [#44431](https://github.com/ClickHouse/ClickHouse/issues/44431): Kill stress tests after 2.5h in case of hanging process. [#44214](https://github.com/ClickHouse/ClickHouse/pull/44214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44557](https://github.com/ClickHouse/ClickHouse/issues/44557): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43512](https://github.com/ClickHouse/ClickHouse/issues/43512): - Fix several buffer over-reads. [#43159](https://github.com/ClickHouse/ClickHouse/pull/43159) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#43750](https://github.com/ClickHouse/ClickHouse/issues/43750): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). @@ -30,4 +30,3 @@ sidebar_label: 2023 #### NO CL ENTRY * NO CL ENTRY: 'Fix multipart upload for large S3 object, backport to 22.3'. [#44217](https://github.com/ClickHouse/ClickHouse/pull/44217) ([ianton-ru](https://github.com/ianton-ru)). - diff --git a/docs/changelogs/v22.3.18.37-lts.md b/docs/changelogs/v22.3.18.37-lts.md index ff6378f09ad..5a9004e7f98 100644 --- a/docs/changelogs/v22.3.18.37-lts.md +++ b/docs/changelogs/v22.3.18.37-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#45856](https://github.com/ClickHouse/ClickHouse/issues/45856): Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45620](https://github.com/ClickHouse/ClickHouse/issues/45620): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45549](https://github.com/ClickHouse/ClickHouse/issues/45549): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -30,4 +30,3 @@ sidebar_label: 2023 * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). - diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md index d5b45f4ce66..dc34ece918d 100644 --- a/docs/changelogs/v22.3.19.6-lts.md +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v22.3.19.6-lts (467e0a7bd77) FIXME as compared to v22.3.18.37-lts (fe512717551) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46440](https://github.com/ClickHouse/ClickHouse/issues/46440): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 3d113d45e68..bf491e46915 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#35928](https://github.com/ClickHouse/ClickHouse/issues/35928): Added settings `input_format_ipv4_default_on_conversion_error`, `input_format_ipv6_default_on_conversion_error` to allow insert of invalid ip address values as default into tables. Closes [#35726](https://github.com/ClickHouse/ClickHouse/issues/35726). [#35733](https://github.com/ClickHouse/ClickHouse/pull/35733) ([Maksim Kita](https://github.com/kitaisreal)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#35415](https://github.com/ClickHouse/ClickHouse/issues/35415): Fix possible deadlock in cache. [#35378](https://github.com/ClickHouse/ClickHouse/pull/35378) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#35563](https://github.com/ClickHouse/ClickHouse/issues/35563): Fix cast into IPv4, IPv6 address in IN section. Fixes [#35528](https://github.com/ClickHouse/ClickHouse/issues/35528). [#35534](https://github.com/ClickHouse/ClickHouse/pull/35534) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.3.4.20-lts.md b/docs/changelogs/v22.3.4.20-lts.md index 72d7b90c743..070ad961beb 100644 --- a/docs/changelogs/v22.3.4.20-lts.md +++ b/docs/changelogs/v22.3.4.20-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * - Add `_le_` method for ClickHouseVersion - Fix auto_version for existing tag - docker_server now support getting version from tags - Add python unit tests to backport workflow. [#36028](https://github.com/ClickHouse/ClickHouse/pull/36028) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36244](https://github.com/ClickHouse/ClickHouse/issues/36244): Fix usage of quota with asynchronous inserts. [#35645](https://github.com/ClickHouse/ClickHouse/pull/35645) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#36240](https://github.com/ClickHouse/ClickHouse/issues/36240): Fix possible loss of subcolumns in type `Object`. [#35682](https://github.com/ClickHouse/ClickHouse/pull/35682) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.5.5-lts.md b/docs/changelogs/v22.3.5.5-lts.md index b4205d5e1a9..4a5dc318b9f 100644 --- a/docs/changelogs/v22.3.5.5-lts.md +++ b/docs/changelogs/v22.3.5.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.5.5-lts (438b4a81f77) FIXME as compared to v22.3.4.20-lts (ecbaf001f49) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36525](https://github.com/ClickHouse/ClickHouse/issues/36525): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36674](https://github.com/ClickHouse/ClickHouse/issues/36674): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 4183332efb0..90e65c7445f 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.6.5-lts (3e44e824cff) FIXME as compared to v22.3.5.5-lts (438b4a81f77) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36795](https://github.com/ClickHouse/ClickHouse/issues/36795): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md index 7347e8e0705..a6a7885abc3 100644 --- a/docs/changelogs/v22.3.7.28-lts.md +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -7,14 +7,14 @@ sidebar_label: 2022 ### ClickHouse release v22.3.7.28-lts (420bdfa2751) FIXME as compared to v22.3.6.5-lts (3e44e824cff) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37715](https://github.com/ClickHouse/ClickHouse/issues/37715): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37511](https://github.com/ClickHouse/ClickHouse/issues/37511): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#37843](https://github.com/ClickHouse/ClickHouse/issues/37843): Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#37940](https://github.com/ClickHouse/ClickHouse/issues/37940): Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#37926](https://github.com/ClickHouse/ClickHouse/issues/37926): Fix check asof join key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37172](https://github.com/ClickHouse/ClickHouse/issues/37172): Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.3.8.39-lts.md b/docs/changelogs/v22.3.8.39-lts.md index 8fff7f00a01..7f9363207d3 100644 --- a/docs/changelogs/v22.3.8.39-lts.md +++ b/docs/changelogs/v22.3.8.39-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38826](https://github.com/ClickHouse/ClickHouse/issues/38826): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38453](https://github.com/ClickHouse/ClickHouse/issues/38453): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#38710](https://github.com/ClickHouse/ClickHouse/issues/38710): Fix incorrect result of distributed queries with `DISTINCT` and `LIMIT`. Fixes [#38282](https://github.com/ClickHouse/ClickHouse/issues/38282). [#38371](https://github.com/ClickHouse/ClickHouse/pull/38371) ([Anton Popov](https://github.com/CurtizJ)). @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#38776](https://github.com/ClickHouse/ClickHouse/issues/38776): `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#38780](https://github.com/ClickHouse/ClickHouse/issues/38780): Fix use-after-free for Map combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36818](https://github.com/ClickHouse/ClickHouse/issues/36818): Fix projection analysis which might lead to wrong query result when IN subquery is used. This fixes [#35336](https://github.com/ClickHouse/ClickHouse/issues/35336). [#35631](https://github.com/ClickHouse/ClickHouse/pull/35631) ([Amos Bird](https://github.com/amosbird)). * Backported in [#38467](https://github.com/ClickHouse/ClickHouse/issues/38467): - Fix potential error with literals in `WHERE` for join queries. Close [#36279](https://github.com/ClickHouse/ClickHouse/issues/36279). [#36542](https://github.com/ClickHouse/ClickHouse/pull/36542) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v22.3.9.19-lts.md b/docs/changelogs/v22.3.9.19-lts.md index 084793f11cc..e0ebcd9b7b8 100644 --- a/docs/changelogs/v22.3.9.19-lts.md +++ b/docs/changelogs/v22.3.9.19-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.9.19-lts (7976930b82e) FIXME as compared to v22.3.8.39-lts (6bcf982f58b) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39097](https://github.com/ClickHouse/ClickHouse/issues/39097): Any allocations inside OvercommitTracker may lead to deadlock. Logging was not very informative so it's easier just to remove logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)). * Backported in [#39080](https://github.com/ClickHouse/ClickHouse/issues/39080): Fix bug in filesystem cache that could happen in some corner case which coincided with cache capacity hitting the limit. Closes [#39066](https://github.com/ClickHouse/ClickHouse/issues/39066). [#39070](https://github.com/ClickHouse/ClickHouse/pull/39070) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index 04ad0bf8955..41f57454d0d 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -161,7 +161,7 @@ sidebar_label: 2022 * call RemoteQueryExecutor with original_query instead of an rewritten query, elimate the AMBIGUOUS_COLUMN_NAME exception. [#35748](https://github.com/ClickHouse/ClickHouse/pull/35748) ([lgbo](https://github.com/lgbo-ustc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Disallow ALTER TTL for engines that does not support it, to avoid breaking ATTACH TABLE (closes [#33344](https://github.com/ClickHouse/ClickHouse/issues/33344)). [#33391](https://github.com/ClickHouse/ClickHouse/pull/33391) ([zhongyuankai](https://github.com/zhongyuankai)). * Do not delay final part writing by default (fixes possible `Memory limit exceeded` during `INSERT` by adding `max_insert_delayed_streams_for_parallel_write` with default to 1000 for writes to s3 and disabled as before otherwise). [#34780](https://github.com/ClickHouse/ClickHouse/pull/34780) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.4.3.3-stable.md b/docs/changelogs/v22.4.3.3-stable.md index 69f95d8cd27..58d0f7b69a1 100644 --- a/docs/changelogs/v22.4.3.3-stable.md +++ b/docs/changelogs/v22.4.3.3-stable.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v22.4.3.3-stable (def956d6299) FIXME as compared to v22.4.2.1-stable (b34ebdc36ae) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36582](https://github.com/ClickHouse/ClickHouse/issues/36582): Fix nullptr dereference in JOIN and COLUMNS matcher. This fixes [#36416](https://github.com/ClickHouse/ClickHouse/issues/36416) . This is for https://github.com/ClickHouse/ClickHouse/pull/36417. [#36430](https://github.com/ClickHouse/ClickHouse/pull/36430) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 1dce0c50e0a..af94ecafcf6 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.4.4.7-stable (ba44414f9b3) FIXME as compared to v22.4.3.3-stable (def956d6299) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36524](https://github.com/ClickHouse/ClickHouse/issues/36524): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36673](https://github.com/ClickHouse/ClickHouse/issues/36673): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index 50cec91b12c..524b309f8ec 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.4.5.9-stable (059ef6cadcd) FIXME as compared to v22.4.4.7-stable (ba44414f9b3) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36635](https://github.com/ClickHouse/ClickHouse/issues/36635): Fix `Missing column` exception which could happen while using `INTERPOLATE` with `ENGINE = MergeTree` table. [#36549](https://github.com/ClickHouse/ClickHouse/pull/36549) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#36794](https://github.com/ClickHouse/ClickHouse/issues/36794): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.4.6.53-stable.md b/docs/changelogs/v22.4.6.53-stable.md index 5dc25697c20..b6380d9c7ba 100644 --- a/docs/changelogs/v22.4.6.53-stable.md +++ b/docs/changelogs/v22.4.6.53-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38828](https://github.com/ClickHouse/ClickHouse/issues/38828): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37717](https://github.com/ClickHouse/ClickHouse/issues/37717): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37512](https://github.com/ClickHouse/ClickHouse/issues/37512): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). @@ -27,7 +27,7 @@ sidebar_label: 2022 * Backported in [#38777](https://github.com/ClickHouse/ClickHouse/issues/38777): `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#38781](https://github.com/ClickHouse/ClickHouse/issues/38781): Fix use-after-free for Map combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#37456](https://github.com/ClickHouse/ClickHouse/issues/37456): Server might fail to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#38468](https://github.com/ClickHouse/ClickHouse/issues/38468): - Fix potential error with literals in `WHERE` for join queries. Close [#36279](https://github.com/ClickHouse/ClickHouse/issues/36279). [#36542](https://github.com/ClickHouse/ClickHouse/pull/36542) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index fdd6325fd1f..28dfa0825cf 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -104,7 +104,7 @@ sidebar_label: 2022 * ClickHouse builds for `PowerPC64LE` architecture are now available in universal installation script `curl https://clickhouse.com/ | sh` and by direct link `https://builds.clickhouse.com/master/powerpc64le/clickhouse`. [#37095](https://github.com/ClickHouse/ClickHouse/pull/37095) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * - Make cmake build scripts a bit more robust. [#37169](https://github.com/ClickHouse/ClickHouse/pull/37169) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * The ilike() function on FixedString columns could have returned wrong results (i.e. match less than it should). [#37117](https://github.com/ClickHouse/ClickHouse/pull/37117) ([Robert Schulze](https://github.com/rschu1ze)). * Fix implicit cast for optimize_skip_unused_shards_rewrite_in. [#37153](https://github.com/ClickHouse/ClickHouse/pull/37153) ([Azat Khuzhin](https://github.com/azat)). @@ -120,7 +120,7 @@ sidebar_label: 2022 * Fix system.opentelemetry_span_log attribute.values alias to values instead of keys. [#37275](https://github.com/ClickHouse/ClickHouse/pull/37275) ([Aleksandr Razumov](https://github.com/ernado)). * Fix possible deadlock in OvercommitTracker during logging. cc @alesapin @tavplubix Fixes [#37272](https://github.com/ClickHouse/ClickHouse/issues/37272). [#37299](https://github.com/ClickHouse/ClickHouse/pull/37299) ([Dmitry Novik](https://github.com/novikd)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * - fix substring function range error length when `offset` and `length` is negative constant and `s` is not constant. [#33861](https://github.com/ClickHouse/ClickHouse/pull/33861) ([RogerYK](https://github.com/RogerYK)). * Accidentally ZSTD support for Arrow was not being built. This fixes [#35283](https://github.com/ClickHouse/ClickHouse/issues/35283). [#35486](https://github.com/ClickHouse/ClickHouse/pull/35486) ([Sean Lafferty](https://github.com/seanlaff)). diff --git a/docs/changelogs/v22.5.2.53-stable.md b/docs/changelogs/v22.5.2.53-stable.md index f2fb50b3e14..8af0c7dac45 100644 --- a/docs/changelogs/v22.5.2.53-stable.md +++ b/docs/changelogs/v22.5.2.53-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38827](https://github.com/ClickHouse/ClickHouse/issues/38827): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37716](https://github.com/ClickHouse/ClickHouse/issues/37716): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37408](https://github.com/ClickHouse/ClickHouse/issues/37408): Throw an exception when GROUPING SETS used with ROLLUP or CUBE. [#37367](https://github.com/ClickHouse/ClickHouse/pull/37367) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.5.3.21-stable.md b/docs/changelogs/v22.5.3.21-stable.md index 994bf79f7a8..4915a41a10c 100644 --- a/docs/changelogs/v22.5.3.21-stable.md +++ b/docs/changelogs/v22.5.3.21-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.5.3.21-stable (e03724efec5) FIXME as compared to v22.5.2.53-stable (5fd600fda9e) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38241](https://github.com/ClickHouse/ClickHouse/issues/38241): Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#39098](https://github.com/ClickHouse/ClickHouse/issues/39098): Any allocations inside OvercommitTracker may lead to deadlock. Logging was not very informative so it's easier just to remove logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.5.4.19-stable.md b/docs/changelogs/v22.5.4.19-stable.md index 24903260904..c90f45c770e 100644 --- a/docs/changelogs/v22.5.4.19-stable.md +++ b/docs/changelogs/v22.5.4.19-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39882](https://github.com/ClickHouse/ClickHouse/issues/39882): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39209](https://github.com/ClickHouse/ClickHouse/issues/39209): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#39589](https://github.com/ClickHouse/ClickHouse/issues/39589): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 0437f682789..c915d24fe00 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -119,7 +119,7 @@ sidebar_label: 2022 * Fix overly aggressive stripping which removed the embedded hash required for checking the consistency of the executable. [#37993](https://github.com/ClickHouse/ClickHouse/pull/37993) ([Robert Schulze](https://github.com/rschu1ze)). * fix MacOS build compressor faild. [#38007](https://github.com/ClickHouse/ClickHouse/pull/38007) ([chen](https://github.com/xiedeyantu)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix `GROUP BY` `AggregateFunction` (i.e. you `GROUP BY` by the column that has `AggregateFunction` type). [#37093](https://github.com/ClickHouse/ClickHouse/pull/37093) ([Azat Khuzhin](https://github.com/azat)). * Fix possible heap-use-after-free error when reading system.projection_parts and system.projection_parts_columns . This fixes [#37184](https://github.com/ClickHouse/ClickHouse/issues/37184). [#37185](https://github.com/ClickHouse/ClickHouse/pull/37185) ([Amos Bird](https://github.com/amosbird)). @@ -169,7 +169,7 @@ sidebar_label: 2022 * when open enable_filesystem_query_cache_limit, throw Reserved cache size exceeds the remaining cache size. [#38004](https://github.com/ClickHouse/ClickHouse/pull/38004) ([chen](https://github.com/xiedeyantu)). * Query, containing ORDER BY ... WITH FILL, can generate extra rows when multiple WITH FILL columns are present. [#38074](https://github.com/ClickHouse/ClickHouse/pull/38074) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix converting types for UNION queries (may produce LOGICAL_ERROR). [#34775](https://github.com/ClickHouse/ClickHouse/pull/34775) ([Azat Khuzhin](https://github.com/azat)). * TTL merge may not be scheduled again if BackgroundExecutor is busy. --merges_with_ttl_counter is increased in selectPartsToMerge() --merge task will be ignored if BackgroundExecutor is busy --merges_with_ttl_counter will not be decrease. [#36387](https://github.com/ClickHouse/ClickHouse/pull/36387) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v22.6.2.12-stable.md b/docs/changelogs/v22.6.2.12-stable.md index d8c1cd31936..3c0f2116f1d 100644 --- a/docs/changelogs/v22.6.2.12-stable.md +++ b/docs/changelogs/v22.6.2.12-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#38484](https://github.com/ClickHouse/ClickHouse/issues/38484): Improve the stability for hive storage integration test. Move the data prepare step into test.py. [#38260](https://github.com/ClickHouse/ClickHouse/pull/38260) ([lgbo](https://github.com/lgbo-ustc)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38404](https://github.com/ClickHouse/ClickHouse/issues/38404): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.6.3.35-stable.md b/docs/changelogs/v22.6.3.35-stable.md index 0a86c136d35..86b8282e075 100644 --- a/docs/changelogs/v22.6.3.35-stable.md +++ b/docs/changelogs/v22.6.3.35-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38883](https://github.com/ClickHouse/ClickHouse/issues/38883): Add `clickhouse-diagnostics` binary to the packages. [#38647](https://github.com/ClickHouse/ClickHouse/pull/38647) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38690](https://github.com/ClickHouse/ClickHouse/issues/38690): Fix incorrect columns order in subqueries of UNION (in case of duplicated columns in subselects may produce incorrect result). [#37887](https://github.com/ClickHouse/ClickHouse/pull/37887) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#38500](https://github.com/ClickHouse/ClickHouse/issues/38500): Do not allow recursive usage of OvercommitTracker during logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794) cc @tavplubix @davenger. [#38246](https://github.com/ClickHouse/ClickHouse/pull/38246) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.6.4.35-stable.md b/docs/changelogs/v22.6.4.35-stable.md index 5c4644f735a..2b5537b5bf9 100644 --- a/docs/changelogs/v22.6.4.35-stable.md +++ b/docs/changelogs/v22.6.4.35-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38822](https://github.com/ClickHouse/ClickHouse/issues/38822): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38242](https://github.com/ClickHouse/ClickHouse/issues/38242): Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#38865](https://github.com/ClickHouse/ClickHouse/issues/38865): Fix s3 seekable reads with parallel read buffer. (Affected memory usage during query). Closes [#38258](https://github.com/ClickHouse/ClickHouse/issues/38258). [#38802](https://github.com/ClickHouse/ClickHouse/pull/38802) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.6.5.22-stable.md b/docs/changelogs/v22.6.5.22-stable.md index 5965f63df14..edb6bdf7c2d 100644 --- a/docs/changelogs/v22.6.5.22-stable.md +++ b/docs/changelogs/v22.6.5.22-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39883](https://github.com/ClickHouse/ClickHouse/issues/39883): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39207](https://github.com/ClickHouse/ClickHouse/issues/39207): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#38932](https://github.com/ClickHouse/ClickHouse/issues/38932): Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.6.6.16-stable.md b/docs/changelogs/v22.6.6.16-stable.md index 4d44621454b..d4d25771326 100644 --- a/docs/changelogs/v22.6.6.16-stable.md +++ b/docs/changelogs/v22.6.6.16-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.6.6.16-stable (d2a33ebc822) FIXME as compared to v22.6.5.22-stable (47ca5f14a34) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40162](https://github.com/ClickHouse/ClickHouse/issues/40162): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40124](https://github.com/ClickHouse/ClickHouse/issues/40124): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.6.7.7-stable.md b/docs/changelogs/v22.6.7.7-stable.md index f5351cc03ed..0b4cc6836f7 100644 --- a/docs/changelogs/v22.6.7.7-stable.md +++ b/docs/changelogs/v22.6.7.7-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40692](https://github.com/ClickHouse/ClickHouse/issues/40692): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40531](https://github.com/ClickHouse/ClickHouse/issues/40531): Proxy resolver stop on first successful request to endpoint. [#40353](https://github.com/ClickHouse/ClickHouse/pull/40353) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#40623](https://github.com/ClickHouse/ClickHouse/issues/40623): Fix potential dataloss due to a bug in AWS SDK (https://github.com/aws/aws-sdk-cpp/issues/658). Bug can be triggered only when clickhouse is used over S3. [#40506](https://github.com/ClickHouse/ClickHouse/pull/40506) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v22.6.8.35-stable.md b/docs/changelogs/v22.6.8.35-stable.md index e68384d3d9c..b69cabffd4d 100644 --- a/docs/changelogs/v22.6.8.35-stable.md +++ b/docs/changelogs/v22.6.8.35-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#41274](https://github.com/ClickHouse/ClickHouse/issues/41274): Fix memory safety issues with functions `encrypt` and `contingency` if Array of Nullable is used as an argument. This fixes [#41004](https://github.com/ClickHouse/ClickHouse/issues/41004). [#40195](https://github.com/ClickHouse/ClickHouse/pull/40195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41282](https://github.com/ClickHouse/ClickHouse/issues/41282): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#40905](https://github.com/ClickHouse/ClickHouse/issues/40905): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.6.9.11-stable.md b/docs/changelogs/v22.6.9.11-stable.md index 5beb9171d9e..42cf0948ecc 100644 --- a/docs/changelogs/v22.6.9.11-stable.md +++ b/docs/changelogs/v22.6.9.11-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41558](https://github.com/ClickHouse/ClickHouse/issues/41558): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41504](https://github.com/ClickHouse/ClickHouse/issues/41504): Writing data in Apache `ORC` format might lead to a buffer overrun. [#41458](https://github.com/ClickHouse/ClickHouse/pull/41458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.7.1.2484-stable.md b/docs/changelogs/v22.7.1.2484-stable.md index 3f90b3691ea..7464b0449ee 100644 --- a/docs/changelogs/v22.7.1.2484-stable.md +++ b/docs/changelogs/v22.7.1.2484-stable.md @@ -128,7 +128,7 @@ sidebar_label: 2022 * Fix LSan by fixing getauxval(). [#39299](https://github.com/ClickHouse/ClickHouse/pull/39299) ([Azat Khuzhin](https://github.com/azat)). * Adapt universal installation script for FreeBSD. [#39302](https://github.com/ClickHouse/ClickHouse/pull/39302) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix projection exception when aggregation keys are wrapped inside other functions. This fixes [#37151](https://github.com/ClickHouse/ClickHouse/issues/37151). [#37155](https://github.com/ClickHouse/ClickHouse/pull/37155) ([Amos Bird](https://github.com/amosbird)). * Fix possible logical error `... with argument with type Nothing and default implementation for Nothing is expected to return result with type Nothing, got ...` in some functions. Closes: [#37610](https://github.com/ClickHouse/ClickHouse/issues/37610) Closes: [#37741](https://github.com/ClickHouse/ClickHouse/issues/37741). [#37759](https://github.com/ClickHouse/ClickHouse/pull/37759) ([Kruglov Pavel](https://github.com/Avogar)). @@ -198,7 +198,7 @@ sidebar_label: 2022 * Fix UB (stack-use-after-scope) in extactAll(). [#39397](https://github.com/ClickHouse/ClickHouse/pull/39397) ([Azat Khuzhin](https://github.com/azat)). * Fix incorrect query result when trivial count optimization is in effect with array join. This fixes [#39431](https://github.com/ClickHouse/ClickHouse/issues/39431). [#39444](https://github.com/ClickHouse/ClickHouse/pull/39444) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Disable send_logs_level for INSERT into Distributed to avoid possible hung. [#35075](https://github.com/ClickHouse/ClickHouse/pull/35075) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.7.2.15-stable.md b/docs/changelogs/v22.7.2.15-stable.md index a9db8bcf10d..0a3748f90f7 100644 --- a/docs/changelogs/v22.7.2.15-stable.md +++ b/docs/changelogs/v22.7.2.15-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#39750](https://github.com/ClickHouse/ClickHouse/issues/39750): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39591](https://github.com/ClickHouse/ClickHouse/issues/39591): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#39613](https://github.com/ClickHouse/ClickHouse/issues/39613): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)). diff --git a/docs/changelogs/v22.7.3.5-stable.md b/docs/changelogs/v22.7.3.5-stable.md index 62a5dfee611..b55b16509d4 100644 --- a/docs/changelogs/v22.7.3.5-stable.md +++ b/docs/changelogs/v22.7.3.5-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#39884](https://github.com/ClickHouse/ClickHouse/issues/39884): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#39884](https://github.com/ClickHouse/ClickHouse/issues/39884): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40045](https://github.com/ClickHouse/ClickHouse/issues/40045): Fix big memory usage during fetches. Fixes [#39915](https://github.com/ClickHouse/ClickHouse/issues/39915). [#39990](https://github.com/ClickHouse/ClickHouse/pull/39990) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#40045](https://github.com/ClickHouse/ClickHouse/issues/40045): Fix big memory usage during fetches. Fixes [#39915](https://github.com/ClickHouse/ClickHouse/issues/39915). [#39990](https://github.com/ClickHouse/ClickHouse/pull/39990) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v22.7.4.16-stable.md b/docs/changelogs/v22.7.4.16-stable.md index 52d68283a2f..4847ef8cf64 100644 --- a/docs/changelogs/v22.7.4.16-stable.md +++ b/docs/changelogs/v22.7.4.16-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.7.4.16-stable (0b9272f8fdc) FIXME as compared to v22.7.3.5-stable (e140b8b5f3a) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40163](https://github.com/ClickHouse/ClickHouse/issues/40163): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40125](https://github.com/ClickHouse/ClickHouse/issues/40125): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.7.5.13-stable.md b/docs/changelogs/v22.7.5.13-stable.md index 0de9e9a26aa..24dbc5c9e7f 100644 --- a/docs/changelogs/v22.7.5.13-stable.md +++ b/docs/changelogs/v22.7.5.13-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40693](https://github.com/ClickHouse/ClickHouse/issues/40693): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40542](https://github.com/ClickHouse/ClickHouse/issues/40542): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#40450](https://github.com/ClickHouse/ClickHouse/issues/40450): Fix rare bug with column TTL for MergeTree engines family: In case of repeated vertical merge the error `Cannot unlink file ColumnName.bin ... No such file or directory.` could happen. [#40346](https://github.com/ClickHouse/ClickHouse/pull/40346) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v22.7.6.74-stable.md b/docs/changelogs/v22.7.6.74-stable.md index 9060375ce8e..3cf2edfccd1 100644 --- a/docs/changelogs/v22.7.6.74-stable.md +++ b/docs/changelogs/v22.7.6.74-stable.md @@ -16,7 +16,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41559](https://github.com/ClickHouse/ClickHouse/issues/41559): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41283](https://github.com/ClickHouse/ClickHouse/issues/41283): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#40865](https://github.com/ClickHouse/ClickHouse/issues/40865): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). diff --git a/docs/changelogs/v22.7.7.24-stable.md b/docs/changelogs/v22.7.7.24-stable.md index cc3a83c5d4c..16e56156ff6 100644 --- a/docs/changelogs/v22.7.7.24-stable.md +++ b/docs/changelogs/v22.7.7.24-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42329](https://github.com/ClickHouse/ClickHouse/issues/42329): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42359](https://github.com/ClickHouse/ClickHouse/issues/42359): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42268](https://github.com/ClickHouse/ClickHouse/issues/42268): Fix reusing of files > 4GB from base backup. [#42146](https://github.com/ClickHouse/ClickHouse/pull/42146) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#42299](https://github.com/ClickHouse/ClickHouse/issues/42299): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.8.1.2097-lts.md b/docs/changelogs/v22.8.1.2097-lts.md index ef454a7e283..b6b634f4826 100644 --- a/docs/changelogs/v22.8.1.2097-lts.md +++ b/docs/changelogs/v22.8.1.2097-lts.md @@ -106,7 +106,7 @@ sidebar_label: 2022 * Support build with `clang-16` (trunk). This closes [#39949](https://github.com/ClickHouse/ClickHouse/issues/39949). [#40181](https://github.com/ClickHouse/ClickHouse/pull/40181) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Prepare RISC-V 64 build to run in CI. This is for [#40141](https://github.com/ClickHouse/ClickHouse/issues/40141). [#40197](https://github.com/ClickHouse/ClickHouse/pull/40197) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fixed query hanging for SELECT with ORDER BY WITH FILL with different date/time types. [#37849](https://github.com/ClickHouse/ClickHouse/pull/37849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Fix ORDER BY that matches projections ORDER BY (before it simply returns unsorted result). [#38725](https://github.com/ClickHouse/ClickHouse/pull/38725) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.8.10.29-lts.md b/docs/changelogs/v22.8.10.29-lts.md index ac41d71650c..33ae27b6da8 100644 --- a/docs/changelogs/v22.8.10.29-lts.md +++ b/docs/changelogs/v22.8.10.29-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43051](https://github.com/ClickHouse/ClickHouse/issues/43051): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43513](https://github.com/ClickHouse/ClickHouse/issues/43513): - Fix several buffer over-reads. [#43159](https://github.com/ClickHouse/ClickHouse/pull/43159) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#43428](https://github.com/ClickHouse/ClickHouse/issues/43428): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.8.11.15-lts.md b/docs/changelogs/v22.8.11.15-lts.md index 337eeba9187..dbe765ca4a2 100644 --- a/docs/changelogs/v22.8.11.15-lts.md +++ b/docs/changelogs/v22.8.11.15-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#43098](https://github.com/ClickHouse/ClickHouse/issues/43098): Updated normaliser to clone the alias ast. resolves [#42452](https://github.com/ClickHouse/ClickHouse/issues/42452) Implementation: * Updated QueryNormalizer to clone alias ast, when its replaced. Previously just assigning the same leads to exception in LogicalExpressinsOptimizer as it would be the same parent being inserted again. * This bug is not seen with new analyser (allow_experimental_analyzer), so no changes for it. I added a test for the same. [#42827](https://github.com/ClickHouse/ClickHouse/pull/42827) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43751](https://github.com/ClickHouse/ClickHouse/issues/43751): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43617](https://github.com/ClickHouse/ClickHouse/issues/43617): Fix sumMap() for Nullable(Decimal()). [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.8.12.45-lts.md b/docs/changelogs/v22.8.12.45-lts.md index 7412784419c..9fab9daeb95 100644 --- a/docs/changelogs/v22.8.12.45-lts.md +++ b/docs/changelogs/v22.8.12.45-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2023 * Backported in [#44378](https://github.com/ClickHouse/ClickHouse/issues/44378): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44558](https://github.com/ClickHouse/ClickHouse/issues/44558): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44751](https://github.com/ClickHouse/ClickHouse/issues/44751): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43525](https://github.com/ClickHouse/ClickHouse/issues/43525): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -39,4 +39,3 @@ sidebar_label: 2023 * Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.8.13.20-lts.md b/docs/changelogs/v22.8.13.20-lts.md index d8dd1bd2b1c..0734f40bf3e 100644 --- a/docs/changelogs/v22.8.13.20-lts.md +++ b/docs/changelogs/v22.8.13.20-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.8.13.20-lts (e4817946d18) FIXME as compared to v22.8.12.45-lts (86b0ecd5d51) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45565](https://github.com/ClickHouse/ClickHouse/issues/45565): Fix positional arguments exception Positional argument out of bounds. Closes [#40634](https://github.com/ClickHouse/ClickHouse/issues/40634). [#41189](https://github.com/ClickHouse/ClickHouse/pull/41189) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#44997](https://github.com/ClickHouse/ClickHouse/issues/44997): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). @@ -21,4 +21,3 @@ sidebar_label: 2023 * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Merge pull request [#38262](https://github.com/ClickHouse/ClickHouse/issues/38262) from PolyProgrammist/fix-ordinary-system-un… [#45650](https://github.com/ClickHouse/ClickHouse/pull/45650) ([alesapin](https://github.com/alesapin)). - diff --git a/docs/changelogs/v22.8.14.53-lts.md b/docs/changelogs/v22.8.14.53-lts.md index 5978080fa3a..3cceb3475b6 100644 --- a/docs/changelogs/v22.8.14.53-lts.md +++ b/docs/changelogs/v22.8.14.53-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#46482](https://github.com/ClickHouse/ClickHouse/issues/46482): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46505](https://github.com/ClickHouse/ClickHouse/issues/46505): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45908](https://github.com/ClickHouse/ClickHouse/issues/45908): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#46238](https://github.com/ClickHouse/ClickHouse/issues/46238): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). @@ -37,4 +37,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.15.23-lts.md b/docs/changelogs/v22.8.15.23-lts.md index 096a504c9c2..5f49dfb1757 100644 --- a/docs/changelogs/v22.8.15.23-lts.md +++ b/docs/changelogs/v22.8.15.23-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2023 #### Bug Fix * Backported in [#47336](https://github.com/ClickHouse/ClickHouse/issues/47336): Sometimes after changing a role that could be not reflected on the access rights of a user who uses that role. This PR fixes that. [#46772](https://github.com/ClickHouse/ClickHouse/pull/46772) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46901](https://github.com/ClickHouse/ClickHouse/issues/46901): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#47156](https://github.com/ClickHouse/ClickHouse/issues/47156): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). @@ -25,4 +25,3 @@ sidebar_label: 2023 * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.3.13-lts.md b/docs/changelogs/v22.8.3.13-lts.md index 903f5b7a600..5f08bc9ee67 100644 --- a/docs/changelogs/v22.8.3.13-lts.md +++ b/docs/changelogs/v22.8.3.13-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40694](https://github.com/ClickHouse/ClickHouse/issues/40694): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40451](https://github.com/ClickHouse/ClickHouse/issues/40451): Fix rare bug with column TTL for MergeTree engines family: In case of repeated vertical merge the error `Cannot unlink file ColumnName.bin ... No such file or directory.` could happen. [#40346](https://github.com/ClickHouse/ClickHouse/pull/40346) ([alesapin](https://github.com/alesapin)). * Backported in [#40533](https://github.com/ClickHouse/ClickHouse/issues/40533): Proxy resolver stop on first successful request to endpoint. [#40353](https://github.com/ClickHouse/ClickHouse/pull/40353) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.8.4.7-lts.md b/docs/changelogs/v22.8.4.7-lts.md index 93d9aa2bf1a..a852120ab8a 100644 --- a/docs/changelogs/v22.8.4.7-lts.md +++ b/docs/changelogs/v22.8.4.7-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.8.4.7-lts (baad27bcd2f) FIXME as compared to v22.8.3.13-lts (6a15b73faea) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40760](https://github.com/ClickHouse/ClickHouse/issues/40760): Fix possible error 'Decimal math overflow' while parsing DateTime64. [#40546](https://github.com/ClickHouse/ClickHouse/pull/40546) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#40811](https://github.com/ClickHouse/ClickHouse/issues/40811): In [#40595](https://github.com/ClickHouse/ClickHouse/issues/40595) it was reported that the `host_regexp` functionality was not working properly with a name to address resolution in `/etc/hosts`. It's fixed. [#40769](https://github.com/ClickHouse/ClickHouse/pull/40769) ([Arthur Passos](https://github.com/arthurpassos)). diff --git a/docs/changelogs/v22.8.5.29-lts.md b/docs/changelogs/v22.8.5.29-lts.md index b7ad3c11a46..1b59501cc28 100644 --- a/docs/changelogs/v22.8.5.29-lts.md +++ b/docs/changelogs/v22.8.5.29-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41157](https://github.com/ClickHouse/ClickHouse/issues/41157): Add macOS binaries to GH release assets, it fixes [#37718](https://github.com/ClickHouse/ClickHouse/issues/37718). [#41088](https://github.com/ClickHouse/ClickHouse/pull/41088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40866](https://github.com/ClickHouse/ClickHouse/issues/40866): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40805](https://github.com/ClickHouse/ClickHouse/issues/40805): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.8.6.71-lts.md b/docs/changelogs/v22.8.6.71-lts.md index 0337c5ba2e2..2ac6ef05bc4 100644 --- a/docs/changelogs/v22.8.6.71-lts.md +++ b/docs/changelogs/v22.8.6.71-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41560](https://github.com/ClickHouse/ClickHouse/issues/41560): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41284](https://github.com/ClickHouse/ClickHouse/issues/41284): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#41837](https://github.com/ClickHouse/ClickHouse/issues/41837): Fix vertical merge of parts with lightweight deleted rows. [#40559](https://github.com/ClickHouse/ClickHouse/pull/40559) ([Alexander Gololobov](https://github.com/davenger)). diff --git a/docs/changelogs/v22.8.7.34-lts.md b/docs/changelogs/v22.8.7.34-lts.md index ee55f5d9f1f..56f03ecdf3b 100644 --- a/docs/changelogs/v22.8.7.34-lts.md +++ b/docs/changelogs/v22.8.7.34-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42296](https://github.com/ClickHouse/ClickHouse/issues/42296): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42360](https://github.com/ClickHouse/ClickHouse/issues/42360): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42489](https://github.com/ClickHouse/ClickHouse/issues/42489): Removed skipping of mutations in unaffected partitions of `MergeTree` tables, because this feature never worked correctly and might cause resurrection of finished mutations. [#40589](https://github.com/ClickHouse/ClickHouse/pull/40589) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#42121](https://github.com/ClickHouse/ClickHouse/issues/42121): Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v22.8.8.3-lts.md b/docs/changelogs/v22.8.8.3-lts.md index b4673eb955a..d42f333ce3f 100644 --- a/docs/changelogs/v22.8.8.3-lts.md +++ b/docs/changelogs/v22.8.8.3-lts.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v22.8.8.3-lts (ac5a6cababc) FIXME as compared to v22.8.7.34-lts (3c38e5e8ab9) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42677](https://github.com/ClickHouse/ClickHouse/issues/42677): keeper-fix: fix race in accessing logs while snapshot is being installed. [#40627](https://github.com/ClickHouse/ClickHouse/pull/40627) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.8.9.24-lts.md b/docs/changelogs/v22.8.9.24-lts.md index 585327b83a1..15935c4cf4e 100644 --- a/docs/changelogs/v22.8.9.24-lts.md +++ b/docs/changelogs/v22.8.9.24-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42964](https://github.com/ClickHouse/ClickHouse/issues/42964): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43040](https://github.com/ClickHouse/ClickHouse/issues/43040): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42720](https://github.com/ClickHouse/ClickHouse/issues/42720): Fixed `Unknown identifier (aggregate-function)` exception which appears when a user tries to calculate WINDOW ORDER BY/PARTITION BY expressions over aggregate functions: ``` CREATE TABLE default.tenk1 ( `unique1` Int32, `unique2` Int32, `ten` Int32 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192; SELECT ten, sum(unique1) + sum(unique2) AS res, rank() OVER (ORDER BY sum(unique1) + sum(unique2) ASC) AS rank FROM _complex GROUP BY ten ORDER BY ten ASC; ``` which gives: ``` Code: 47. DB::Exception: Received from localhost:9000. DB::Exception: Unknown identifier: sum(unique1); there are columns: unique1, unique2, ten: While processing sum(unique1) + sum(unique2) ASC. (UNKNOWN_IDENTIFIER) ```. [#39762](https://github.com/ClickHouse/ClickHouse/pull/39762) ([Vladimir Chebotaryov](https://github.com/quickhouse)). * Backported in [#42748](https://github.com/ClickHouse/ClickHouse/issues/42748): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). diff --git a/docs/changelogs/v22.9.2.7-stable.md b/docs/changelogs/v22.9.2.7-stable.md index 5c4db4bfa96..bbd0a6cce32 100644 --- a/docs/changelogs/v22.9.2.7-stable.md +++ b/docs/changelogs/v22.9.2.7-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#41709](https://github.com/ClickHouse/ClickHouse/issues/41709): Check file path for path traversal attacks in errors logger for input formats. [#41694](https://github.com/ClickHouse/ClickHouse/pull/41694) ([Kruglov Pavel](https://github.com/Avogar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41696](https://github.com/ClickHouse/ClickHouse/issues/41696): Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). diff --git a/docs/changelogs/v22.9.3.18-stable.md b/docs/changelogs/v22.9.3.18-stable.md index 656cb1dfc22..a46dba6718c 100644 --- a/docs/changelogs/v22.9.3.18-stable.md +++ b/docs/changelogs/v22.9.3.18-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.9.3.18-stable (0cb4b15d2fa) FIXME as compared to v22.9.2.7-stable (362e2cefcef) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41902](https://github.com/ClickHouse/ClickHouse/issues/41902): Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#41863](https://github.com/ClickHouse/ClickHouse/issues/41863): 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v22.9.4.32-stable.md b/docs/changelogs/v22.9.4.32-stable.md index 658d39af079..92bcc01e408 100644 --- a/docs/changelogs/v22.9.4.32-stable.md +++ b/docs/changelogs/v22.9.4.32-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42297](https://github.com/ClickHouse/ClickHouse/issues/42297): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42361](https://github.com/ClickHouse/ClickHouse/issues/42361): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42122](https://github.com/ClickHouse/ClickHouse/issues/42122): Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#41938](https://github.com/ClickHouse/ClickHouse/issues/41938): Don't allow to create or alter merge tree tables with virtual column name _row_exists, which is reserved for lightweight delete. Fixed [#41716](https://github.com/ClickHouse/ClickHouse/issues/41716). [#41763](https://github.com/ClickHouse/ClickHouse/pull/41763) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.9.5.25-stable.md b/docs/changelogs/v22.9.5.25-stable.md index eb46fb893e7..90150726ace 100644 --- a/docs/changelogs/v22.9.5.25-stable.md +++ b/docs/changelogs/v22.9.5.25-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42965](https://github.com/ClickHouse/ClickHouse/issues/42965): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43041](https://github.com/ClickHouse/ClickHouse/issues/43041): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42749](https://github.com/ClickHouse/ClickHouse/issues/42749): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#42863](https://github.com/ClickHouse/ClickHouse/issues/42863): Fix lowerUTF8()/upperUTF8() in case of symbol was in between 16-byte boundary (very frequent case of you have strings > 16 bytes long). [#42812](https://github.com/ClickHouse/ClickHouse/pull/42812) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.9.6.20-stable.md b/docs/changelogs/v22.9.6.20-stable.md index d450f285848..7abc4adc32e 100644 --- a/docs/changelogs/v22.9.6.20-stable.md +++ b/docs/changelogs/v22.9.6.20-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43052](https://github.com/ClickHouse/ClickHouse/issues/43052): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43505](https://github.com/ClickHouse/ClickHouse/issues/43505): Fix a bug when row level filter uses default value of column. [#43387](https://github.com/ClickHouse/ClickHouse/pull/43387) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43722](https://github.com/ClickHouse/ClickHouse/issues/43722): Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). diff --git a/docs/changelogs/v22.9.7.34-stable.md b/docs/changelogs/v22.9.7.34-stable.md index 83be449f70d..ed8173eaf50 100644 --- a/docs/changelogs/v22.9.7.34-stable.md +++ b/docs/changelogs/v22.9.7.34-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#44111](https://github.com/ClickHouse/ClickHouse/issues/44111): Bring sha512 sums back to the building step. [#44017](https://github.com/ClickHouse/ClickHouse/pull/44017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43612](https://github.com/ClickHouse/ClickHouse/issues/43612): Fix bad inefficiency of `remote_filesystem_read_method=read` with filesystem cache. Closes [#42125](https://github.com/ClickHouse/ClickHouse/issues/42125). [#42129](https://github.com/ClickHouse/ClickHouse/pull/42129) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#43526](https://github.com/ClickHouse/ClickHouse/issues/43526): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v23.1.1.3077-stable.md b/docs/changelogs/v23.1.1.3077-stable.md index e218be62f09..53ca9e1831c 100644 --- a/docs/changelogs/v23.1.1.3077-stable.md +++ b/docs/changelogs/v23.1.1.3077-stable.md @@ -125,7 +125,7 @@ sidebar_label: 2023 * SQLite library is updated to the latest. It is used for the SQLite database and table integration engines. Also, fixed a false-positive TSan report. This closes [#45027](https://github.com/ClickHouse/ClickHouse/issues/45027). [#45031](https://github.com/ClickHouse/ClickHouse/pull/45031) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix report sending in the case when FastTest failed. [#45588](https://github.com/ClickHouse/ClickHouse/pull/45588) ([Dmitry Novik](https://github.com/novikd)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * #40651 [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Fix possible use-of-unitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) CC: @nickitat. [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)). @@ -589,4 +589,3 @@ sidebar_label: 2023 * Resubmit "Fix possible in-use table after DETACH" [#45566](https://github.com/ClickHouse/ClickHouse/pull/45566) ([Alexander Tokmakov](https://github.com/tavplubix)). * Typo: "Granulesis" --> "Granules" [#45598](https://github.com/ClickHouse/ClickHouse/pull/45598) ([Robert Schulze](https://github.com/rschu1ze)). * Fix version in autogenerated_versions.txt [#45624](https://github.com/ClickHouse/ClickHouse/pull/45624) ([Dmitry Novik](https://github.com/novikd)). - diff --git a/docs/changelogs/v23.1.2.9-stable.md b/docs/changelogs/v23.1.2.9-stable.md index 272a2b95a86..7d34a6b9ec2 100644 --- a/docs/changelogs/v23.1.2.9-stable.md +++ b/docs/changelogs/v23.1.2.9-stable.md @@ -13,11 +13,10 @@ sidebar_label: 2023 #### Bug Fix * Backported in [#45673](https://github.com/ClickHouse/ClickHouse/issues/45673): Fix wiping sensitive info in logs. [#45603](https://github.com/ClickHouse/ClickHouse/pull/45603) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45730](https://github.com/ClickHouse/ClickHouse/issues/45730): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.3.5-stable.md b/docs/changelogs/v23.1.3.5-stable.md index d4f39894bec..9f8ef928138 100644 --- a/docs/changelogs/v23.1.3.5-stable.md +++ b/docs/changelogs/v23.1.3.5-stable.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v23.1.3.5-stable (548b494bcce) FIXME as compared to v23.1.2.9-stable (8dfb1700858) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45896](https://github.com/ClickHouse/ClickHouse/issues/45896): Bugfix IPv6 parser for mixed ip4 address with missed first octet (like `::.1.2.3`). [#45871](https://github.com/ClickHouse/ClickHouse/pull/45871) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.4.58-stable.md b/docs/changelogs/v23.1.4.58-stable.md index d1ffe87f58e..9081d700308 100644 --- a/docs/changelogs/v23.1.4.58-stable.md +++ b/docs/changelogs/v23.1.4.58-stable.md @@ -20,7 +20,7 @@ sidebar_label: 2023 * Backported in [#46477](https://github.com/ClickHouse/ClickHouse/issues/46477): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46511](https://github.com/ClickHouse/ClickHouse/issues/46511): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46228](https://github.com/ClickHouse/ClickHouse/issues/46228): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#46967](https://github.com/ClickHouse/ClickHouse/issues/46967): Backward compatibility - allow implicit narrowing conversion from UInt64 to IPv4 - required for "INSERT ... VALUES ..." expression. [#45865](https://github.com/ClickHouse/ClickHouse/pull/45865) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -44,4 +44,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.5.24-stable.md b/docs/changelogs/v23.1.5.24-stable.md index 1c2c127a8c3..934e97312c0 100644 --- a/docs/changelogs/v23.1.5.24-stable.md +++ b/docs/changelogs/v23.1.5.24-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#47060](https://github.com/ClickHouse/ClickHouse/issues/47060): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46401](https://github.com/ClickHouse/ClickHouse/issues/46401): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). * Backported in [#46905](https://github.com/ClickHouse/ClickHouse/issues/46905): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). @@ -25,4 +25,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.1.2537-stable.md b/docs/changelogs/v23.2.1.2537-stable.md index 3fdcf6d6571..9da81c039ea 100644 --- a/docs/changelogs/v23.2.1.2537-stable.md +++ b/docs/changelogs/v23.2.1.2537-stable.md @@ -161,7 +161,7 @@ sidebar_label: 2023 * Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Raised the minimum Clang version needed to build ClickHouse from 12 to 15. [#46710](https://github.com/ClickHouse/ClickHouse/pull/46710) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Flush data exactly by `rabbitmq_flush_interval_ms` or by `rabbitmq_max_block_size` in `StorageRabbitMQ`. Closes [#42389](https://github.com/ClickHouse/ClickHouse/issues/42389). Closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#44404](https://github.com/ClickHouse/ClickHouse/pull/44404) ([Kseniia Sumarokova](https://github.com/kssenii)). * - Use PODArray to render in sparkBar function, so we can control the memory usage. Close [#44467](https://github.com/ClickHouse/ClickHouse/issues/44467). [#44489](https://github.com/ClickHouse/ClickHouse/pull/44489) ([Duc Canh Le](https://github.com/canhld94)). @@ -470,4 +470,3 @@ sidebar_label: 2023 #### Testing Improvement * Fixed functional test 00304_http_external_data for s390x. [#45807](https://github.com/ClickHouse/ClickHouse/pull/45807) ([Harry Lee](https://github.com/HarryLeeIBM)). - diff --git a/docs/changelogs/v23.2.2.20-stable.md b/docs/changelogs/v23.2.2.20-stable.md index 60aeaa66cbf..b92fbdebe33 100644 --- a/docs/changelogs/v23.2.2.20-stable.md +++ b/docs/changelogs/v23.2.2.20-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#47062](https://github.com/ClickHouse/ClickHouse/issues/47062): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46895](https://github.com/ClickHouse/ClickHouse/issues/46895): Fixed a bug in automatic retries of `DROP TABLE` query with `ReplicatedMergeTree` tables and `Atomic` databases. In rare cases it could lead to `Can't get data for node /zk_path/log_pointer` and `The specified key does not exist` errors if ZooKeeper session expired during DROP and a new replicated table with the same path in ZooKeeper was created in parallel. [#46384](https://github.com/ClickHouse/ClickHouse/pull/46384) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#46865](https://github.com/ClickHouse/ClickHouse/issues/46865): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -27,4 +27,3 @@ sidebar_label: 2023 * More concise logging at trace level for PREWHERE steps [#46771](https://github.com/ClickHouse/ClickHouse/pull/46771) ([Alexander Gololobov](https://github.com/davenger)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.3.17-stable.md b/docs/changelogs/v23.2.3.17-stable.md index fb2c4e394dc..75b7f8b2b20 100644 --- a/docs/changelogs/v23.2.3.17-stable.md +++ b/docs/changelogs/v23.2.3.17-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v23.2.3.17-stable (dec18bf7281) FIXME as compared to v23.2.2.20-stable (f6c269c8df2) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46907](https://github.com/ClickHouse/ClickHouse/issues/46907): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#47091](https://github.com/ClickHouse/ClickHouse/issues/47091): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). @@ -20,4 +20,3 @@ sidebar_label: 2023 * Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * do flushUntrackedMemory when context switches [#47102](https://github.com/ClickHouse/ClickHouse/pull/47102) ([Sema Checherinda](https://github.com/CheSema)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.4.12-stable.md b/docs/changelogs/v23.2.4.12-stable.md index 2b6a689aee5..1542e3257ce 100644 --- a/docs/changelogs/v23.2.4.12-stable.md +++ b/docs/changelogs/v23.2.4.12-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v23.2.4.12-stable (8fe866cb035) FIXME as compared to v23.2.3.17-stable (dec18bf7281) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#47277](https://github.com/ClickHouse/ClickHouse/issues/47277): Fix IPv4/IPv6 serialization/deserialization in binary formats that was broken in https://github.com/ClickHouse/ClickHouse/pull/43221. Closes [#46522](https://github.com/ClickHouse/ClickHouse/issues/46522). [#46616](https://github.com/ClickHouse/ClickHouse/pull/46616) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#47212](https://github.com/ClickHouse/ClickHouse/issues/47212): `INSERT` queries through native TCP protocol and HTTP protocol were not canceled correctly in some cases. It could lead to a partially applied query if a client canceled the query, or if a client died or, in rare cases, on network errors. As a result, it could lead to not working deduplication. Fixes [#27667](https://github.com/ClickHouse/ClickHouse/issues/27667) and [#45377](https://github.com/ClickHouse/ClickHouse/issues/45377). [#46681](https://github.com/ClickHouse/ClickHouse/pull/46681) ([Alexander Tokmakov](https://github.com/tavplubix)). @@ -17,4 +17,3 @@ sidebar_label: 2023 * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 41574907492..8dc136e01f4 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -35,8 +35,8 @@ LABELS = { "pr-backward-incompatible": ["Backward Incompatible Change"], "pr-bugfix": [ "Bug Fix", - "Bug Fix (user-visible misbehaviour in official stable or prestable release)", - "Bug Fix (user-visible misbehavior in official stable or prestable release)", + "Bug Fix (user-visible misbehaviour in official stable release)", + "Bug Fix (user-visible misbehavior in official stable release)", ], "pr-build": [ "Build/Testing/Packaging Improvement", diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 3b66b68193b..9972ce8a627 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -256,6 +256,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri category = "NOT FOR CHANGELOG / INSIGNIFICANT" return Description(item.number, item.user, item.html_url, item.title, category) + # Normalize bug fixes + if re.match( + r"(?i)bug\Wfix", + category, + ): + category = "Bug Fix (user-visible misbehavior in an official stable release)" + return Description(item.number, item.user, item.html_url, item.title, category) + # Filter out documentations changelog if re.match( r"(?i)doc", From a71454f8ec53d56ca4332220843ea0540dcafca4 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:53:31 +0200 Subject: [PATCH 36/73] Fix automatic indentation in the built-in UI SQL editor --- programs/server/play.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index a7fc951ec37..323ba7d0b9e 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -687,7 +687,7 @@ e.preventDefault(); return false; - } else if (e.key === 'Enter') { + } else if (e.key === 'Enter' && !(event.metaKey || event.ctrlKey)) { // If the user presses Enter, and the previous line starts with spaces, // then we will insert the same number of spaces. const elem = e.target; From 65c496176963fe6c04b73c4232f38602c65f9cf1 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:47:35 +0200 Subject: [PATCH 37/73] Update tests/queries/0_stateless/25402_fail_on_invalid_having.sql --- tests/queries/0_stateless/25402_fail_on_invalid_having.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/25402_fail_on_invalid_having.sql b/tests/queries/0_stateless/25402_fail_on_invalid_having.sql index beb72218685..2d46a335dc1 100644 --- a/tests/queries/0_stateless/25402_fail_on_invalid_having.sql +++ b/tests/queries/0_stateless/25402_fail_on_invalid_having.sql @@ -1 +1 @@ -SELECT a, sum(b) FROM (SELECT 1 AS a, 1 AS b, 0 AS c) GROUP BY a HAVING c -- {{ serverError NOT_AN_AGGREGATE}} \ No newline at end of file +SELECT a, sum(b) FROM (SELECT 1 AS a, 1 AS b, 0 AS c) GROUP BY a HAVING c SETTINGS allow_experimental_analyzer=1 -- {{ serverError NOT_AN_AGGREGATE}} \ No newline at end of file From 08fa1846b1e0f23f4714edc28e5aedf98d2ca094 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 27 Mar 2023 11:11:42 +0000 Subject: [PATCH 38/73] Add merge tree setting max_number_of_mutations_for_replica --- src/Core/ServerSettings.h | 2 +- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 5 +++++ src/Storages/MergeTree/MergeTreeSettings.h | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 4fc721e70a2..e780424507c 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -61,7 +61,7 @@ namespace DB M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ M(UInt64, background_pool_size, 16, "The maximum number of threads what will be used for merging or mutating data parts for *MergeTree-engine tables in a background.", 0) \ - M(UInt64, background_merges_mutations_concurrency_ratio, 2, "The multiplier which shows the relation between the number of tasks that could be executed concurrently and the number of threads being used.", 0) \ + M(UInt64, background_merges_mutations_concurrency_ratio, 2, "The number of part mutation tasks that can be executed concurrently by each thread in background pool.", 0) \ M(String, background_merges_mutations_scheduling_policy, "round_robin", "The policy on how to perform a scheduling for background merges and mutations. Possible values are: `round_robin` and `shortest_task_first`. ", 0) \ M(UInt64, background_move_pool_size, 8, "The maximum number of threads that will be used for moving data parts to another disk or volume for *MergeTree-engine tables in a background.", 0) \ M(UInt64, background_fetches_pool_size, 8, "The maximum number of threads that will be used for fetching data parts from another replica for *MergeTree-engine tables in a background.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index cf009a10c27..49289df6161 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -113,9 +113,14 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation() const const auto data_settings = data.getSettings(); size_t occupied = CurrentMetrics::values[CurrentMetrics::BackgroundMergesAndMutationsPoolTask].load(std::memory_order_relaxed); + if (data_settings->max_number_of_mutations_for_replica > 0 && + occupied >= data_settings->max_number_of_mutations_for_replica) + return 0; + /// DataPart can be store only at one disk. Get maximum reservable free space at all disks. UInt64 disk_space = data.getStoragePolicy()->getMaxUnreservedFreeSpace(); auto max_tasks_count = data.getContext()->getMergeMutateExecutor()->getMaxTasksCount(); + /// Allow mutations only if there are enough threads, leave free threads for merges else if (occupied <= 1 || max_tasks_count - occupied >= data_settings->number_of_free_entries_in_pool_to_execute_mutation) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 6d280f3aaec..4aabcbbf865 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -48,6 +48,7 @@ struct Settings; M(UInt64, max_replicated_merges_with_ttl_in_queue, 1, "How many tasks of merging parts with TTL are allowed simultaneously in ReplicatedMergeTree queue.", 0) \ M(UInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.", 0) \ M(UInt64, number_of_free_entries_in_pool_to_execute_mutation, 20, "When there is less than specified number of free entries in pool, do not execute part mutations. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \ + M(UInt64, max_number_of_mutations_for_replica, 0, "Limit the number of part mutations per replica to the specified amount. Zero means no limit on the number of mutations per replica (the execution can still be constrained by other settings).", 0) \ M(UInt64, max_number_of_merges_with_ttl_in_pool, 2, "When there is more than specified number of merges with TTL entries in pool, do not assign new merge with TTL. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \ M(Seconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.", 0) \ M(Seconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories. You should not lower this value because merges and mutations may not be able to work with low value of this setting.", 0) \ From cfd10dafe47b07a573e15015c9aa1b907e9fce62 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 27 Mar 2023 09:22:07 -0300 Subject: [PATCH 39/73] only link arrow lib if it exists, same for ut --- src/CMakeLists.txt | 4 +++- .../tests/gtest_assert_arrow_log_does_not_abort.cpp | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5cdd14d826..62005c76ce0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -608,7 +608,6 @@ if (ENABLE_TESTS) dbms clickhouse_common_config clickhouse_common_zookeeper - ch_contrib::parquet string_utils) if (TARGET ch_contrib::simdjson) @@ -623,6 +622,9 @@ if (ENABLE_TESTS) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::azure_sdk) endif() + if (TARGET ch_contrib::parquet) + target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::parquet) + endif() add_check(unit_tests_dbms) endif () diff --git a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp index 203f2d7cef6..8152ced287e 100644 --- a/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp +++ b/src/Processors/tests/gtest_assert_arrow_log_does_not_abort.cpp @@ -1,3 +1,7 @@ +#include "config.h" + +#if USE_ARROW || USE_PARQUET + #include #include #include @@ -19,3 +23,5 @@ TEST(ArrowLog, FatalLogShouldThrow) } } + +#endif From 4234c38a64a0ba7a7c2c152592b7f857813b30f1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 14:45:49 +0200 Subject: [PATCH 40/73] Automatically correct some mistakes in the changelog --- docs/changelogs/v22.10.5.54-stable.md | 1 - docs/changelogs/v22.10.6.3-stable.md | 1 - docs/changelogs/v22.10.7.13-stable.md | 1 - docs/changelogs/v22.11.3.47-stable.md | 1 - docs/changelogs/v22.11.4.3-stable.md | 1 - docs/changelogs/v22.11.5.15-stable.md | 1 - docs/changelogs/v22.11.6.44-stable.md | 1 - docs/changelogs/v22.12.3.5-stable.md | 1 - docs/changelogs/v22.12.4.76-stable.md | 1 - docs/changelogs/v22.12.5.34-stable.md | 1 - docs/changelogs/v22.3.16.1190-lts.md | 1 - docs/changelogs/v22.3.18.37-lts.md | 1 - docs/changelogs/v22.3.19.6-lts.md | 1 - docs/changelogs/v22.8.12.45-lts.md | 1 - docs/changelogs/v22.8.13.20-lts.md | 1 - docs/changelogs/v22.8.14.53-lts.md | 1 - docs/changelogs/v22.8.15.23-lts.md | 1 - docs/changelogs/v23.1.1.3077-stable.md | 1 - docs/changelogs/v23.1.2.9-stable.md | 1 - docs/changelogs/v23.1.3.5-stable.md | 1 - docs/changelogs/v23.1.4.58-stable.md | 1 - docs/changelogs/v23.1.5.24-stable.md | 1 - docs/changelogs/v23.2.1.2537-stable.md | 1 - docs/changelogs/v23.2.2.20-stable.md | 1 - docs/changelogs/v23.2.3.17-stable.md | 1 - docs/changelogs/v23.2.4.12-stable.md | 1 - utils/changelog/changelog.py | 20 ++++++++++++++++++-- 27 files changed, 18 insertions(+), 28 deletions(-) diff --git a/docs/changelogs/v22.10.5.54-stable.md b/docs/changelogs/v22.10.5.54-stable.md index e372fb30618..aa1173dc671 100644 --- a/docs/changelogs/v22.10.5.54-stable.md +++ b/docs/changelogs/v22.10.5.54-stable.md @@ -41,4 +41,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.10.6.3-stable.md b/docs/changelogs/v22.10.6.3-stable.md index b0e88c92cb0..43ca032c774 100644 --- a/docs/changelogs/v22.10.6.3-stable.md +++ b/docs/changelogs/v22.10.6.3-stable.md @@ -10,4 +10,3 @@ sidebar_label: 2023 #### Bug Fix (user-visible misbehavior in official stable or prestable release) * Backported in [#45084](https://github.com/ClickHouse/ClickHouse/issues/45084): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.10.7.13-stable.md b/docs/changelogs/v22.10.7.13-stable.md index c906e00e524..46e876f36b4 100644 --- a/docs/changelogs/v22.10.7.13-stable.md +++ b/docs/changelogs/v22.10.7.13-stable.md @@ -18,4 +18,3 @@ sidebar_label: 2023 * Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.3.47-stable.md b/docs/changelogs/v22.11.3.47-stable.md index d6451b853f7..832591e68f3 100644 --- a/docs/changelogs/v22.11.3.47-stable.md +++ b/docs/changelogs/v22.11.3.47-stable.md @@ -37,4 +37,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.11.4.3-stable.md b/docs/changelogs/v22.11.4.3-stable.md index 33780e848ef..fbf2baa9f7b 100644 --- a/docs/changelogs/v22.11.4.3-stable.md +++ b/docs/changelogs/v22.11.4.3-stable.md @@ -10,4 +10,3 @@ sidebar_label: 2023 #### Bug Fix (user-visible misbehavior in official stable or prestable release) * Backported in [#45085](https://github.com/ClickHouse/ClickHouse/issues/45085): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.11.5.15-stable.md b/docs/changelogs/v22.11.5.15-stable.md index 742a8740514..6927dccf7b7 100644 --- a/docs/changelogs/v22.11.5.15-stable.md +++ b/docs/changelogs/v22.11.5.15-stable.md @@ -19,4 +19,3 @@ sidebar_label: 2023 * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.6.44-stable.md b/docs/changelogs/v22.11.6.44-stable.md index 6e628b85150..cb7b2667667 100644 --- a/docs/changelogs/v22.11.6.44-stable.md +++ b/docs/changelogs/v22.11.6.44-stable.md @@ -34,4 +34,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.3.5-stable.md b/docs/changelogs/v22.12.3.5-stable.md index 8cbcbc6a590..2770de71f0c 100644 --- a/docs/changelogs/v22.12.3.5-stable.md +++ b/docs/changelogs/v22.12.3.5-stable.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.12.4.76-stable.md b/docs/changelogs/v22.12.4.76-stable.md index 79569ff841e..7f4334c565b 100644 --- a/docs/changelogs/v22.12.4.76-stable.md +++ b/docs/changelogs/v22.12.4.76-stable.md @@ -52,4 +52,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.5.34-stable.md b/docs/changelogs/v22.12.5.34-stable.md index 95befaa88ff..11613bd838b 100644 --- a/docs/changelogs/v22.12.5.34-stable.md +++ b/docs/changelogs/v22.12.5.34-stable.md @@ -26,4 +26,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.3.16.1190-lts.md b/docs/changelogs/v22.3.16.1190-lts.md index a43d34551ca..5f38dc32c16 100644 --- a/docs/changelogs/v22.3.16.1190-lts.md +++ b/docs/changelogs/v22.3.16.1190-lts.md @@ -30,4 +30,3 @@ sidebar_label: 2023 #### NO CL ENTRY * NO CL ENTRY: 'Fix multipart upload for large S3 object, backport to 22.3'. [#44217](https://github.com/ClickHouse/ClickHouse/pull/44217) ([ianton-ru](https://github.com/ianton-ru)). - diff --git a/docs/changelogs/v22.3.18.37-lts.md b/docs/changelogs/v22.3.18.37-lts.md index ff6378f09ad..65c05b35149 100644 --- a/docs/changelogs/v22.3.18.37-lts.md +++ b/docs/changelogs/v22.3.18.37-lts.md @@ -30,4 +30,3 @@ sidebar_label: 2023 * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). - diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md index d5b45f4ce66..1cc4b9cc013 100644 --- a/docs/changelogs/v22.3.19.6-lts.md +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.12.45-lts.md b/docs/changelogs/v22.8.12.45-lts.md index 7412784419c..7fde9de07f2 100644 --- a/docs/changelogs/v22.8.12.45-lts.md +++ b/docs/changelogs/v22.8.12.45-lts.md @@ -39,4 +39,3 @@ sidebar_label: 2023 * Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.8.13.20-lts.md b/docs/changelogs/v22.8.13.20-lts.md index d8dd1bd2b1c..a2ed41fc97d 100644 --- a/docs/changelogs/v22.8.13.20-lts.md +++ b/docs/changelogs/v22.8.13.20-lts.md @@ -21,4 +21,3 @@ sidebar_label: 2023 * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Merge pull request [#38262](https://github.com/ClickHouse/ClickHouse/issues/38262) from PolyProgrammist/fix-ordinary-system-un… [#45650](https://github.com/ClickHouse/ClickHouse/pull/45650) ([alesapin](https://github.com/alesapin)). - diff --git a/docs/changelogs/v22.8.14.53-lts.md b/docs/changelogs/v22.8.14.53-lts.md index 5978080fa3a..0b5170953eb 100644 --- a/docs/changelogs/v22.8.14.53-lts.md +++ b/docs/changelogs/v22.8.14.53-lts.md @@ -37,4 +37,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.15.23-lts.md b/docs/changelogs/v22.8.15.23-lts.md index 096a504c9c2..faa5dcd9860 100644 --- a/docs/changelogs/v22.8.15.23-lts.md +++ b/docs/changelogs/v22.8.15.23-lts.md @@ -25,4 +25,3 @@ sidebar_label: 2023 * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.1.3077-stable.md b/docs/changelogs/v23.1.1.3077-stable.md index e218be62f09..025d1242b11 100644 --- a/docs/changelogs/v23.1.1.3077-stable.md +++ b/docs/changelogs/v23.1.1.3077-stable.md @@ -589,4 +589,3 @@ sidebar_label: 2023 * Resubmit "Fix possible in-use table after DETACH" [#45566](https://github.com/ClickHouse/ClickHouse/pull/45566) ([Alexander Tokmakov](https://github.com/tavplubix)). * Typo: "Granulesis" --> "Granules" [#45598](https://github.com/ClickHouse/ClickHouse/pull/45598) ([Robert Schulze](https://github.com/rschu1ze)). * Fix version in autogenerated_versions.txt [#45624](https://github.com/ClickHouse/ClickHouse/pull/45624) ([Dmitry Novik](https://github.com/novikd)). - diff --git a/docs/changelogs/v23.1.2.9-stable.md b/docs/changelogs/v23.1.2.9-stable.md index 272a2b95a86..825481d8382 100644 --- a/docs/changelogs/v23.1.2.9-stable.md +++ b/docs/changelogs/v23.1.2.9-stable.md @@ -20,4 +20,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.3.5-stable.md b/docs/changelogs/v23.1.3.5-stable.md index d4f39894bec..bd4a59ac512 100644 --- a/docs/changelogs/v23.1.3.5-stable.md +++ b/docs/changelogs/v23.1.3.5-stable.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.4.58-stable.md b/docs/changelogs/v23.1.4.58-stable.md index d1ffe87f58e..035c55863a0 100644 --- a/docs/changelogs/v23.1.4.58-stable.md +++ b/docs/changelogs/v23.1.4.58-stable.md @@ -44,4 +44,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.5.24-stable.md b/docs/changelogs/v23.1.5.24-stable.md index 1c2c127a8c3..f490b92e725 100644 --- a/docs/changelogs/v23.1.5.24-stable.md +++ b/docs/changelogs/v23.1.5.24-stable.md @@ -25,4 +25,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.1.2537-stable.md b/docs/changelogs/v23.2.1.2537-stable.md index 3fdcf6d6571..7feafe7af65 100644 --- a/docs/changelogs/v23.2.1.2537-stable.md +++ b/docs/changelogs/v23.2.1.2537-stable.md @@ -470,4 +470,3 @@ sidebar_label: 2023 #### Testing Improvement * Fixed functional test 00304_http_external_data for s390x. [#45807](https://github.com/ClickHouse/ClickHouse/pull/45807) ([Harry Lee](https://github.com/HarryLeeIBM)). - diff --git a/docs/changelogs/v23.2.2.20-stable.md b/docs/changelogs/v23.2.2.20-stable.md index 60aeaa66cbf..ae41fb24100 100644 --- a/docs/changelogs/v23.2.2.20-stable.md +++ b/docs/changelogs/v23.2.2.20-stable.md @@ -27,4 +27,3 @@ sidebar_label: 2023 * More concise logging at trace level for PREWHERE steps [#46771](https://github.com/ClickHouse/ClickHouse/pull/46771) ([Alexander Gololobov](https://github.com/davenger)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.3.17-stable.md b/docs/changelogs/v23.2.3.17-stable.md index fb2c4e394dc..1735c0e5925 100644 --- a/docs/changelogs/v23.2.3.17-stable.md +++ b/docs/changelogs/v23.2.3.17-stable.md @@ -20,4 +20,3 @@ sidebar_label: 2023 * Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * do flushUntrackedMemory when context switches [#47102](https://github.com/ClickHouse/ClickHouse/pull/47102) ([Sema Checherinda](https://github.com/CheSema)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.4.12-stable.md b/docs/changelogs/v23.2.4.12-stable.md index 2b6a689aee5..071fde68298 100644 --- a/docs/changelogs/v23.2.4.12-stable.md +++ b/docs/changelogs/v23.2.4.12-stable.md @@ -17,4 +17,3 @@ sidebar_label: 2023 * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 3b66b68193b..6b70952eced 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -126,8 +126,8 @@ def get_descriptions(prs: PullRequests) -> Dict[str, List[Description]]: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Generate a changelog in MD format between given tags. " - "It fetches all tags and unshallow the git repositore automatically", + description="Generate a changelog in Markdown format between given tags. " + "It fetches all tags and unshallow the git repository automatically", ) parser.add_argument( "-v", @@ -243,6 +243,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri else: i += 1 + # Remove excessive bullets from the entry. + if re.match(r"^[\-\*] ", entry): + entry = entry[2:] + + # Better style. + if re.match(r"^[a-z]", entry): + entry = entry.capitalize() + if not category: # Shouldn't happen, because description check in CI should catch such PRs. # Fall through, so that it shows up in output and the user can fix it. @@ -256,6 +264,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri category = "NOT FOR CHANGELOG / INSIGNIFICANT" return Description(item.number, item.user, item.html_url, item.title, category) + # Normalize bug fixes + if re.match( + r"(?i)bug\Wfix", + category, + ): + category = "Bug Fix (user-visible misbehavior in an official stable release)" + return Description(item.number, item.user, item.html_url, item.title, category) + # Filter out documentations changelog if re.match( r"(?i)doc", From 64db4c478b76e179d067806795a3353d10372410 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 27 Mar 2023 13:19:45 +0000 Subject: [PATCH 41/73] Use one map --- src/Interpreters/ProcessList.cpp | 19 ++++++++++--------- src/Interpreters/ProcessList.h | 5 ++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index a86377a37d5..d66d4bdea64 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -362,7 +362,7 @@ QueryStatus::~QueryStatus() { #if !defined(NDEBUG) /// Check that all executors were invalidated. - for (const auto & e : executors) + for (const auto & [_, e] : executors) assert(!e->executor); #endif @@ -400,7 +400,9 @@ CancellationCode QueryStatus::cancelQuery(bool) { /// Create a snapshot of executors under a mutex. std::lock_guard lock(executors_mutex); - executors_snapshot = executors; + executors_snapshot.reserve(executors.size()); + for (const auto & [_, e] : executors) + executors_snapshot.push_back(e); } /// We should call cancel() for each executor with unlocked executors_mutex, because @@ -428,9 +430,8 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(!executor_indexes.contains(e)); - executors.push_back(std::make_shared(e)); - executor_indexes[e] = executors.size() - 1; + assert(!executors.contains(e)); + executors[e] = std::make_shared(e); } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) @@ -439,12 +440,12 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { std::lock_guard lock(executors_mutex); - assert(executor_indexes.contains(e)); - executor_holder = executors[executor_indexes[e]]; - executor_indexes.erase(e); + assert(executors.contains(e)); + executor_holder = executors[e]; + executors.erase(e); } - /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) + /// Invalidate executor pointer inside holder. /// We should do it with released executors_mutex to avoid possible lock order inversion. executor_holder->remove(); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 9e5c91bf2ed..a04beac4901 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -133,9 +133,8 @@ protected: using ExecutorHolderPtr = std::shared_ptr; - /// Array of PipelineExecutors to be cancelled when a cancelQuery is received - std::vector executors; - std::unordered_map executor_indexes; + /// Container of PipelineExecutors to be cancelled when a cancelQuery is received + std::unordered_map executors; enum QueryStreamsStatus { From 1845bfc6446f033475ae18799e20af4daface821 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 14:13:32 +0200 Subject: [PATCH 42/73] Revert change to ld64.lld linker on Mac This reverts PRs #42470, #47673 and #47744. The problem was that after the switch to ld64.lld, server binaries build in Debug mode no longer came up, see the discussion after (*). My attempt to fix it with `-no_compact_unwind` didn't help. I also tried - `-keep_dwarf_unwind`, - `-unwindlib` (to use the OS unwinder), - the unwinder in contrib (CMake variable EXCEPTION_HANDLING_LIBRARY) but w/o success. Just tons of wasted time. Rolling back to lld as linker on Mac. This will bring back many linker warnings (#42282) but I rather accept that (temporarily, maybe someone can figure out how to fix them) than have a broken Debug binary. (*) https://github.com/ClickHouse/ClickHouse/pull/42470#issuecomment-1312344068 --- CMakeLists.txt | 14 ++------------ cmake/tools.cmake | 6 +----- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5550a19b699..bd7de46474b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ endif () if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. - if (LINKER_NAME MATCHES "lld" AND OS_LINUX) + if (LINKER_NAME MATCHES "lld") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") message (STATUS "Adding .gdb-index via --gdb-index linker option.") endif () @@ -212,7 +212,7 @@ endif () # Create BuildID when using lld. For other linkers it is created by default. # (NOTE: LINKER_NAME can be either path or name, and in different variants) -if (LINKER_NAME MATCHES "lld" AND OS_LINUX) +if (LINKER_NAME MATCHES "lld") # SHA1 is not cryptographically secure but it is the best what lld is offering. set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") endif () @@ -349,17 +349,7 @@ set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_I if (COMPILER_CLANG) if (OS_DARWIN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main") - - # The LLVM MachO linker (ld64.lld, used in native builds) generates by default unwind info in 'compact' format which the internal - # unwinder doesn't support and the server will not come up ('invalid compact unwind encoding'). Disable it. You will see warning - # during the build "ld64.lld: warning: Option `-no_compact_unwind' is undocumented. Should lld implement it?". Yes, ld64.lld does - # not document the option, likely for compat with Apple's system ld after which ld64.lld is modeled after and which also does not - # document it. - if (NOT CMAKE_CROSSCOMPILING) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_compact_unwind") - endif () endif() # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 974b0bd1d3d..3572134d89f 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -61,17 +61,13 @@ if (NOT LINKER_NAME) if (COMPILER_GCC) find_program (LLD_PATH NAMES "ld.lld") elseif (COMPILER_CLANG) - # llvm lld is a generic driver. - # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead if (OS_LINUX) if (NOT ARCH_S390X) # s390x doesnt support lld find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld") endif () - elseif (OS_DARWIN) - find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld") endif () endif () - if (OS_LINUX OR OS_DARWIN) + if (OS_LINUX) if (LLD_PATH) if (COMPILER_GCC) # GCC driver requires one of supported linker names like "lld". From cb04c6301c51ebf92aef27fc70bede97cae710b3 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 27 Mar 2023 16:30:53 +0200 Subject: [PATCH 43/73] =?UTF-8?q?Support=20for=20cte=20in=20parameterized?= =?UTF-8?q?=20views=E2=80=A8Implementation:=20*=20Updated=20to=20allow=20q?= =?UTF-8?q?uery=20parameters=20while=20evaluating=20scalar=20subqueries.?= =?UTF-8?q?=20Testing:=20*=20Added=20test=20case=20with=20cute=20for=20par?= =?UTF-8?q?ameterized=20view.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExecuteScalarSubqueriesVisitor.cpp | 1 + .../ExecuteScalarSubqueriesVisitor.h | 1 + src/Interpreters/TreeRewriter.cpp | 8 +++---- .../02428_parameterized_view.reference | 1 + .../0_stateless/02428_parameterized_view.sh | 22 ++++++++++++++++++- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 7a5fc67596f..3d5f14f4723 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -98,6 +98,7 @@ static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqu ASTPtr subquery_select = subquery.children.at(0); auto options = SelectQueryOptions(QueryProcessingStage::Complete, data.subquery_depth + 1, true); + options.is_create_parameterized_view = data.is_create_parameterized_view; options.analyze(data.only_analyze); return std::make_unique(subquery_select, subquery_context, options); diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h index d702404dab6..577140f731f 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h @@ -36,6 +36,7 @@ public: Scalars & scalars; Scalars & local_scalars; bool only_analyze; + bool is_create_parameterized_view; }; static bool needChildVisit(ASTPtr & node, const ASTPtr &); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 716d63f03e4..bf684077942 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -452,10 +452,10 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const /// Replacing scalar subqueries with constant values. void executeScalarSubqueries( - ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, Scalars & local_scalars, bool only_analyze) + ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, Scalars & local_scalars, bool only_analyze, bool is_create_parameterized_view) { LogAST log; - ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, local_scalars, only_analyze}; + ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, local_scalars, only_analyze, is_create_parameterized_view}; ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); } @@ -1222,7 +1222,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates); /// Executing scalar subqueries - replacing them with constant values. - executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze); + executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze, select_options.is_create_parameterized_view); if (settings.legacy_column_name_of_tuple_literal) markTupleLiteralsAsLegacy(query); @@ -1336,7 +1336,7 @@ TreeRewriterResultPtr TreeRewriter::analyze( normalize(query, result.aliases, result.source_columns_set, false, settings, allow_self_aliases, getContext(), is_create_parameterized_view); /// Executing scalar subqueries. Column defaults could be a scalar subquery. - executeScalarSubqueries(query, getContext(), 0, result.scalars, result.local_scalars, !execute_scalar_subqueries); + executeScalarSubqueries(query, getContext(), 0, result.scalars, result.local_scalars, !execute_scalar_subqueries, is_create_parameterized_view); if (settings.legacy_column_name_of_tuple_literal) markTupleLiteralsAsLegacy(query); diff --git a/tests/queries/0_stateless/02428_parameterized_view.reference b/tests/queries/0_stateless/02428_parameterized_view.reference index 5af9e75680c..59a3f4d67dd 100644 --- a/tests/queries/0_stateless/02428_parameterized_view.reference +++ b/tests/queries/0_stateless/02428_parameterized_view.reference @@ -38,3 +38,4 @@ ERROR 10 10 10 +1 diff --git a/tests/queries/0_stateless/02428_parameterized_view.sh b/tests/queries/0_stateless/02428_parameterized_view.sh index 274ee43e00d..57a43ce08a0 100755 --- a/tests/queries/0_stateless/02428_parameterized_view.sh +++ b/tests/queries/0_stateless/02428_parameterized_view.sh @@ -23,6 +23,9 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog" $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}" +$CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS 02428_trace_view" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02428_otel_traces_trace_id_ts" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02428_otel_traces" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_02428_Catalog (Name String, Price UInt64, Quantity UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parameterized_view', 'r1') ORDER BY Name" @@ -95,6 +98,20 @@ $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv11(price=10)" $CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv12 AS SELECT * from ( SELECT Price FROM test_02428_Catalog WHERE Price IN (SELECT number FROM numbers({price:UInt64})) )" $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv12(price=11)" +$CLICKHOUSE_CLIENT -q "CREATE TABLE 02428_otel_traces (TraceId String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/otel_traces', 'r1') ORDER BY TraceId" +$CLICKHOUSE_CLIENT -q "CREATE TABLE 02428_otel_traces_trace_id_ts (TraceId String, Start Timestamp) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/otel_traces_trace_id_ts', 'r1') ORDER BY TraceId" + +$CLICKHOUSE_CLIENT -q "INSERT INTO 02428_otel_traces(TraceId) VALUES ('1')" +$CLICKHOUSE_CLIENT -q "INSERT INTO 02428_otel_traces_trace_id_ts(TraceId, Start) VALUES('1', now())" + +$CLICKHOUSE_CLIENT -q "CREATE VIEW 02428_trace_view AS WITH {trace_id:String} AS trace_id, + ( SELECT min(Start) FROM 02428_otel_traces_trace_id_ts WHERE TraceId = trace_id + ) AS start SELECT + TraceId AS traceID + FROM 02428_otel_traces" +$CLICKHOUSE_CLIENT -q "SELECT * FROM 02428_trace_view(trace_id='1')" + + $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv1" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv2" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv3" @@ -110,4 +127,7 @@ $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_v1" $CLICKHOUSE_CLIENT -q "DROP TABLE test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" $CLICKHOUSE_CLIENT -q "DROP TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog" -$CLICKHOUSE_CLIENT -q "DROP DATABASE ${CLICKHOUSE_TEST_UNIQUE_NAME}" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP DATABASE ${CLICKHOUSE_TEST_UNIQUE_NAME}" +$CLICKHOUSE_CLIENT -q "DROP VIEW 02428_trace_view" +$CLICKHOUSE_CLIENT -q "DROP TABLE 02428_otel_traces_trace_id_ts" +$CLICKHOUSE_CLIENT -q "DROP TABLE 02428_otel_traces" \ No newline at end of file From 61d41e8152cc79166de54c11e10ccf50b1418fe4 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 27 Mar 2023 14:41:36 +0000 Subject: [PATCH 44/73] Use uniq names for Records in Avro to avoid reusing its schema --- src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 4 +++- .../02592_avro_records_with_same_names.reference | 1 + .../0_stateless/02592_avro_records_with_same_names.sh | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02592_avro_records_with_same_names.reference create mode 100755 tests/queries/0_stateless/02592_avro_records_with_same_names.sh diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index f1b42147cd6..8a851b511da 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -386,7 +386,9 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF const auto & nested_names = tuple_type.getElementNames(); std::vector nested_serializers; nested_serializers.reserve(nested_types.size()); - auto schema = avro::RecordSchema(column_name); + /// We should use unique names for records. Otherwise avro will reuse schema of this record later + /// for all records with the same name. + auto schema = avro::RecordSchema(column_name + "_" + std::to_string(type_name_increment)); for (size_t i = 0; i != nested_types.size(); ++i) { auto nested_mapping = createSchemaWithSerializeFn(nested_types[i], type_name_increment, nested_names[i]); diff --git a/tests/queries/0_stateless/02592_avro_records_with_same_names.reference b/tests/queries/0_stateless/02592_avro_records_with_same_names.reference new file mode 100644 index 00000000000..7237be8884e --- /dev/null +++ b/tests/queries/0_stateless/02592_avro_records_with_same_names.reference @@ -0,0 +1 @@ +((1,2)) ((3,4,5)) diff --git a/tests/queries/0_stateless/02592_avro_records_with_same_names.sh b/tests/queries/0_stateless/02592_avro_records_with_same_names.sh new file mode 100755 index 00000000000..d13d36230e3 --- /dev/null +++ b/tests/queries/0_stateless/02592_avro_records_with_same_names.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select tuple(tuple(1, 2))::Tuple(x Tuple(a UInt32, b UInt32)) as c1, tuple(tuple(3, 4, 5))::Tuple(x Tuple(c UInt32, d UInt32, e UInt32)) as c2 format Avro" | $CLICKHOUSE_LOCAL --input-format Avro --structure 'c1 Tuple(x Tuple(a UInt32, b UInt32)), c2 Tuple(x Tuple(c UInt32, d UInt32, e UInt32))' -q "select * from table" From 3701fda6b1ab05542117f7773ffd05e6dc4154ba Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Mar 2023 13:43:15 +0000 Subject: [PATCH 45/73] Add str_to_date() alias for MySQL compat Fixes #43755 --- .../functions/type-conversion-functions.md | 18 +++ src/Functions/parseDateTime.cpp | 135 +++++++++++++++--- ...new_functions_must_be_documented.reference | 4 + .../02668_parse_datetime.reference | 13 ++ .../0_stateless/02668_parse_datetime.sql | 10 +- ...68_parse_datetime_in_joda_syntax.reference | 9 ++ .../02668_parse_datetime_in_joda_syntax.sql | 6 + 7 files changed, 172 insertions(+), 23 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 90f6cf0aa7d..6e4686a0b93 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1188,6 +1188,16 @@ SELECT parseDateTime('2021-01-04+23:00:00', '%Y-%m-%d+%H:%i:%s') Alias: `TO_TIMESTAMP`. +## parseDateTimeOrZero + +Same as for [parseDateTime](#type_conversion_functions-parseDateTime) except that it returns zero date when it encounters a date format that cannot be processed. + +## parseDateTimeOrNull + +Same as for [parseDateTime](#type_conversion_functions-parseDateTime) except that it returns `NULL` when it encounters a date format that cannot be processed. + +Alias: `str_to_date`. + ## parseDateTimeInJodaSyntax {#type_conversion_functions-parseDateTimeInJodaSyntax} Similar to [parseDateTime](#parsedatetime), except that the format string is in [Joda](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) instead of MySQL syntax. @@ -1227,6 +1237,14 @@ SELECT parseDateTimeInJodaSyntax('2023-02-24 14:53:31', 'yyyy-MM-dd HH:mm:ss', ' └─────────────────────────────────────────────────────────────────────────────────────────┘ ``` +## parseDateTimeInJodaSyntaxOrZero + +Same as for [parseDateTimeInJodaSyntax](#type_conversion_functions-parseDateTimeInJodaSyntax) except that it returns zero date when it encounters a date format that cannot be processed. + +## parseDateTimeInJodaSyntaxOrNull + +Same as for [parseDateTimeInJodaSyntax](#type_conversion_functions-parseDateTimeInJodaSyntax) except that it returns `NULL` when it encounters a date format that cannot be processed. + ## parseDateTimeBestEffort ## parseDateTime32BestEffort diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index abee7e0d8f8..6a7a6010d4b 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -452,8 +454,15 @@ namespace Joda }; + enum class ErrorHandling + { + Exception, + Zero, + Null + }; + /// _FUNC_(str[, format, timezone]) - template + template class FunctionParseDateTimeImpl : public IFunction { public: @@ -500,11 +509,14 @@ namespace getName()); String time_zone_name = getTimeZone(arguments).getTimeZone(); - return std::make_shared(time_zone_name); + DataTypePtr date_type = std::make_shared(time_zone_name); + if (error_handling == ErrorHandling::Null) + return std::make_shared(date_type); + else + return date_type; } - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { const auto * col_str = checkAndGetColumn(arguments[0].column.get()); if (!col_str) @@ -518,8 +530,12 @@ namespace const auto & time_zone = getTimeZone(arguments); std::vector instructions = parseFormat(format); - auto col_res = ColumnDateTime::create(); - col_res->reserve(input_rows_count); + auto col_res = ColumnDateTime::create(input_rows_count); + + ColumnUInt8::MutablePtr col_null_map; + if constexpr (error_handling == ErrorHandling::Null) + col_null_map = ColumnUInt8::create(input_rows_count, 0); + auto & res_data = col_res->getData(); /// Make datetime fit in a cache line. @@ -527,29 +543,77 @@ namespace for (size_t i = 0; i < input_rows_count; ++i) { datetime.reset(); - StringRef str_ref = col_str->getDataAt(i); Pos cur = str_ref.data; Pos end = str_ref.data + str_ref.size; + bool error = false; + for (const auto & instruction : instructions) { - cur = instruction.perform(cur, end, datetime); + try + { + cur = instruction.perform(cur, end, datetime); + } + catch (...) + { + if constexpr (error_handling == ErrorHandling::Zero) + { + res_data[i] = 0; + error = true; + break; + } + else if constexpr (error_handling == ErrorHandling::Null) + { + res_data[i] = 0; + col_null_map->getData()[i] = 1; + error = true; + break; + } + else + { + static_assert(error_handling == ErrorHandling::Exception); + throw; + } + } } - // Ensure all input was consumed. - if (cur < end) - throw Exception( - ErrorCodes::CANNOT_PARSE_DATETIME, - "Invalid format input {} is malformed at {}", - str_ref.toView(), - std::string_view(cur, end - cur)); + if (error) + continue; - Int64 time = datetime.buildDateTime(time_zone); - res_data.push_back(static_cast(time)); + try + { + /// Ensure all input was consumed + if (cur < end) + throw Exception( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Invalid format input {} is malformed at {}", + str_ref.toView(), + std::string_view(cur, end - cur)); + Int64 time = datetime.buildDateTime(time_zone); + res_data[i] = static_cast(time); + } + catch (...) + { + if constexpr (error_handling == ErrorHandling::Zero) + res_data[i] = 0; + else if constexpr (error_handling == ErrorHandling::Null) + { + res_data[i] = 0; + col_null_map->getData()[i] = 1; + } + else + { + static_assert(error_handling == ErrorHandling::Exception); + throw; + } + } } - return col_res; - } + if constexpr (error_handling == ErrorHandling::Null) + return ColumnNullable::create(std::move(col_res), std::move(col_null_map)); + else + return col_res; + } private: @@ -1753,23 +1817,50 @@ namespace static constexpr auto name = "parseDateTime"; }; + struct NameParseDateTimeOrZero + { + static constexpr auto name = "parseDateTimeOrZero"; + }; + + struct NameParseDateTimeOrNull + { + static constexpr auto name = "parseDateTimeOrNull"; + }; + struct NameParseDateTimeInJodaSyntax { static constexpr auto name = "parseDateTimeInJodaSyntax"; }; + struct NameParseDateTimeInJodaSyntaxOrZero + { + static constexpr auto name = "parseDateTimeInJodaSyntaxOrZero"; + }; - using FunctionParseDateTime = FunctionParseDateTimeImpl; - using FunctionParseDateTimeInJodaSyntax - = FunctionParseDateTimeImpl; + struct NameParseDateTimeInJodaSyntaxOrNull + { + static constexpr auto name = "parseDateTimeInJodaSyntaxOrNull"; + }; + + using FunctionParseDateTime = FunctionParseDateTimeImpl; + using FunctionParseDateTimeOrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTimeOrNull = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntax = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntaxOrZero = FunctionParseDateTimeImpl; + using FunctionParseDateTimeInJodaSyntaxOrNull = FunctionParseDateTimeImpl; } REGISTER_FUNCTION(ParseDateTime) { factory.registerFunction(); factory.registerAlias("TO_UNIXTIME", FunctionParseDateTime::name); + factory.registerFunction(); + factory.registerFunction(); + factory.registerAlias("str_to_date", FunctionParseDateTimeOrNull::name); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 4ae98bda16d..6682f5668ea 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -530,6 +530,10 @@ parseDateTimeBestEffortUS parseDateTimeBestEffortUSOrNull parseDateTimeBestEffortUSOrZero parseDateTimeInJodaSyntax +parseDateTimeInJodaSyntaxOrNull +parseDateTimeInJodaSyntaxOrZero +parseDateTimeOrNull +parseDateTimeOrZero parseTimeDelta partitionId path diff --git a/tests/queries/0_stateless/02668_parse_datetime.reference b/tests/queries/0_stateless/02668_parse_datetime.reference index b893c1bc6e9..f39655c6a41 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.reference +++ b/tests/queries/0_stateless/02668_parse_datetime.reference @@ -197,3 +197,16 @@ select parseDateTime('2019-07-03 11:04:10', '%Y-%m-%d %H:%i:%s', 'UTC') = toDate 1 select parseDateTime('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); 1 +-- *OrZero, *OrNull, str_to_date +select parseDateTimeOrZero('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +1 +select parseDateTimeOrZero('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('1970-01-01 00:00:00', 'UTC'); +1 +select parseDateTimeOrNull('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +1 +select parseDateTimeOrNull('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL; +1 +select str_to_date('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +1 +select str_to_date('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL; +1 diff --git a/tests/queries/0_stateless/02668_parse_datetime.sql b/tests/queries/0_stateless/02668_parse_datetime.sql index 86e8877eedc..757c4fe2efe 100644 --- a/tests/queries/0_stateless/02668_parse_datetime.sql +++ b/tests/queries/0_stateless/02668_parse_datetime.sql @@ -132,4 +132,12 @@ select parseDateTime('2021-01-04+23:00:00', '%Y-%m-%d+%H:%i:%s', 'UTC') = toDate select parseDateTime('2019-07-03 11:04:10', '%Y-%m-%d %H:%i:%s', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); select parseDateTime('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); --- { echoOff } \ No newline at end of file +-- *OrZero, *OrNull, str_to_date +select parseDateTimeOrZero('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +select parseDateTimeOrZero('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('1970-01-01 00:00:00', 'UTC'); +select parseDateTimeOrNull('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +select parseDateTimeOrNull('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL; +select str_to_date('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTime('2019-07-03 11:04:10', 'UTC'); +select str_to_date('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL; + +-- { echoOff } diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference index 7402f104ae4..124836d6118 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference @@ -344,3 +344,12 @@ select parseDateTimeInJodaSyntax('-1', 's', 'UTC'); -- { serverError CANNOT_PARS select parseDateTimeInJodaSyntax('123456789', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } -- integer overflow in AST Fuzzer select parseDateTimeInJodaSyntax('19191919191919191919191919191919', 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } +-- *OrZero, *OrNull +select parseDateTimeInJodaSyntaxOrZero('2001 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('2000-12-31', 'UTC'); +1 +select parseDateTimeInJodaSyntaxOrZero('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('1970-01-01', 'UTC'); +1 +select parseDateTimeInJodaSyntaxOrNull('2001 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('2000-12-31', 'UTC'); +1 +select parseDateTimeInJodaSyntaxOrNull('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') IS NULL; +1 diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql index 7ce5c1a4fdd..b2c781432d1 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql @@ -232,4 +232,10 @@ select parseDateTimeInJodaSyntax('123456789', 's', 'UTC'); -- { serverError CANN -- integer overflow in AST Fuzzer select parseDateTimeInJodaSyntax('19191919191919191919191919191919', 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } +-- *OrZero, *OrNull +select parseDateTimeInJodaSyntaxOrZero('2001 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('2000-12-31', 'UTC'); +select parseDateTimeInJodaSyntaxOrZero('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('1970-01-01', 'UTC'); +select parseDateTimeInJodaSyntaxOrNull('2001 366 2000', 'yyyy D yyyy', 'UTC') = toDateTime('2000-12-31', 'UTC'); +select parseDateTimeInJodaSyntaxOrNull('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') IS NULL; + -- { echoOff } From f1258a932fcaf34e5a6241501ffc9d5519f1fe72 Mon Sep 17 00:00:00 2001 From: nellicus Date: Mon, 27 Mar 2023 16:58:20 +0200 Subject: [PATCH 46/73] fix typo --- docs/en/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index fa44e6e2978..ee2336e9378 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -120,7 +120,7 @@ Query: CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; ``` -## LDAP Exernal User Directory {#ldap-external-user-directory} +## LDAP External User Directory {#ldap-external-user-directory} In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. To achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. From eac71397c4bb68b20bb071582ba33b71de84c60d Mon Sep 17 00:00:00 2001 From: rfraposa Date: Mon, 27 Mar 2023 09:19:50 -0600 Subject: [PATCH 47/73] Create marked_dropped_tables.md --- .../system-tables/marked_dropped_tables.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/en/operations/system-tables/marked_dropped_tables.md diff --git a/docs/en/operations/system-tables/marked_dropped_tables.md b/docs/en/operations/system-tables/marked_dropped_tables.md new file mode 100644 index 00000000000..a260aea1c5e --- /dev/null +++ b/docs/en/operations/system-tables/marked_dropped_tables.md @@ -0,0 +1,37 @@ +--- +slug: /en/operations/system-tables/marked_dropped_tables +--- +# marked_dropped_tables + +Contains information about tables that drop table has been executed but data cleanup has not been actually performed. + +Columns: + +- `index` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Index in marked_dropped_tables queue. +- `database` ([String](../../sql-reference/data-types/string.md)) — Database. +- `table` ([String](../../sql-reference/data-types/string.md)) — Table name. +- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid. +- `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name. +- `metadata_dropped_path` ([String](../../sql-reference/data-types/string.md)) — Path of table's metadata file in metadate_dropped directory. +- `table_dropped_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — The time when the next attempt to remove table's data is scheduled on. Usually it's the table when the table was dropped plus `database_atomic_delay_before_drop_table_sec` + +**Example** + +The following example shows how to get information about marked_dropped_tables. + +``` sql +SELECT * +FROM system.marked_dropped_tables\G +``` + +``` text +Row 1: +────── +index: 0 +database: default +table: test +uuid: 03141bb2-e97a-4d7c-a172-95cc066bb3bd +engine: MergeTree +metadata_dropped_path: /data/ClickHouse/build/programs/data/metadata_dropped/default.test.03141bb2-e97a-4d7c-a172-95cc066bb3bd.sql +table_dropped_time: 2023-03-16 23:43:31 +``` \ No newline at end of file From 246316dd49c5eca803e67b6018eea03756f3b4e1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 15:23:30 +0000 Subject: [PATCH 48/73] Docs: Document [FULL] keyword in SHOW TABLES Follow-up to #43910 --- docs/en/sql-reference/statements/show.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index a9f0aedccdf..7d5e2845c4a 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -117,7 +117,7 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" Displays a list of tables. ```sql -SHOW [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` If the `FROM` clause is not specified, the query returns the list of tables from the current database. @@ -521,4 +521,4 @@ Outputs the content of the [system.table_engines](../../operations/system-tables **See Also** -- [system.table_engines](../../operations/system-tables/table_engines.md) table \ No newline at end of file +- [system.table_engines](../../operations/system-tables/table_engines.md) table From 15039d33271acfb93c393c3165a72cdf3beac259 Mon Sep 17 00:00:00 2001 From: rfraposa Date: Mon, 27 Mar 2023 09:36:56 -0600 Subject: [PATCH 49/73] Rename file --- .../system-tables/{marked_dropped_tables.md => dropped_tables.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/en/operations/system-tables/{marked_dropped_tables.md => dropped_tables.md} (100%) diff --git a/docs/en/operations/system-tables/marked_dropped_tables.md b/docs/en/operations/system-tables/dropped_tables.md similarity index 100% rename from docs/en/operations/system-tables/marked_dropped_tables.md rename to docs/en/operations/system-tables/dropped_tables.md From ceefe77ce2395f4522498a4012486efba8b960f4 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 27 Mar 2023 17:40:17 +0200 Subject: [PATCH 50/73] Fix style --- tests/queries/0_stateless/02592_avro_records_with_same_names.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02592_avro_records_with_same_names.sh b/tests/queries/0_stateless/02592_avro_records_with_same_names.sh index d13d36230e3..92a7846d3bd 100755 --- a/tests/queries/0_stateless/02592_avro_records_with_same_names.sh +++ b/tests/queries/0_stateless/02592_avro_records_with_same_names.sh @@ -2,7 +2,6 @@ # Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh From 1b63b0b043b0cd53569329d5089d29836f2200f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 27 Mar 2023 17:51:22 +0200 Subject: [PATCH 51/73] Fix stateless tests numbers --- ..._bug.reference => 02695_logical_optimizer_alias_bug.reference} | 0 ...imizer_alias_bug.sql => 02695_logical_optimizer_alias_bug.sql} | 0 ...erence => 02695_storage_join_insert_select_deadlock.reference} | 0 ...deadlock.sql => 02695_storage_join_insert_select_deadlock.sql} | 0 ...checksums.reference => 02696_inverted_idx_checksums.reference} | 0 ...nverted_idx_checksums.sql => 02696_inverted_idx_checksums.sql} | 0 ...cel.reference => 02697_stop_reading_on_first_cancel.reference} | 0 ...g_on_first_cancel.sh => 02697_stop_reading_on_first_cancel.sh} | 0 ...ped_tables.reference => 02698_marked_dropped_tables.reference} | 0 ..._marked_dropped_tables.sql => 02698_marked_dropped_tables.sql} | 0 ...p.reference => 02699_polygons_sym_difference_rollup.reference} | 0 ...erence_rollup.sql => 02699_polygons_sym_difference_rollup.sql} | 0 ..._regexp_operator.reference => 02700_regexp_operator.reference} | 0 .../{25401_regexp_operator.sql => 02700_regexp_operator.sql} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{25340_logical_optimizer_alias_bug.reference => 02695_logical_optimizer_alias_bug.reference} (100%) rename tests/queries/0_stateless/{25340_logical_optimizer_alias_bug.sql => 02695_logical_optimizer_alias_bug.sql} (100%) rename tests/queries/0_stateless/{25340_storage_join_insert_select_deadlock.reference => 02695_storage_join_insert_select_deadlock.reference} (100%) rename tests/queries/0_stateless/{25340_storage_join_insert_select_deadlock.sql => 02695_storage_join_insert_select_deadlock.sql} (100%) rename tests/queries/0_stateless/{25341_inverted_idx_checksums.reference => 02696_inverted_idx_checksums.reference} (100%) rename tests/queries/0_stateless/{25341_inverted_idx_checksums.sql => 02696_inverted_idx_checksums.sql} (100%) rename tests/queries/0_stateless/{25341_stop_reading_on_first_cancel.reference => 02697_stop_reading_on_first_cancel.reference} (100%) rename tests/queries/0_stateless/{25341_stop_reading_on_first_cancel.sh => 02697_stop_reading_on_first_cancel.sh} (100%) rename tests/queries/0_stateless/{25400_marked_dropped_tables.reference => 02698_marked_dropped_tables.reference} (100%) rename tests/queries/0_stateless/{25400_marked_dropped_tables.sql => 02698_marked_dropped_tables.sql} (100%) rename tests/queries/0_stateless/{25401_polygons_sym_difference_rollup.reference => 02699_polygons_sym_difference_rollup.reference} (100%) rename tests/queries/0_stateless/{25401_polygons_sym_difference_rollup.sql => 02699_polygons_sym_difference_rollup.sql} (100%) rename tests/queries/0_stateless/{25401_regexp_operator.reference => 02700_regexp_operator.reference} (100%) rename tests/queries/0_stateless/{25401_regexp_operator.sql => 02700_regexp_operator.sql} (100%) diff --git a/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference b/tests/queries/0_stateless/02695_logical_optimizer_alias_bug.reference similarity index 100% rename from tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference rename to tests/queries/0_stateless/02695_logical_optimizer_alias_bug.reference diff --git a/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql b/tests/queries/0_stateless/02695_logical_optimizer_alias_bug.sql similarity index 100% rename from tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql rename to tests/queries/0_stateless/02695_logical_optimizer_alias_bug.sql diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference b/tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.reference similarity index 100% rename from tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference rename to tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.reference diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql b/tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.sql similarity index 100% rename from tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql rename to tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.sql diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.reference b/tests/queries/0_stateless/02696_inverted_idx_checksums.reference similarity index 100% rename from tests/queries/0_stateless/25341_inverted_idx_checksums.reference rename to tests/queries/0_stateless/02696_inverted_idx_checksums.reference diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.sql b/tests/queries/0_stateless/02696_inverted_idx_checksums.sql similarity index 100% rename from tests/queries/0_stateless/25341_inverted_idx_checksums.sql rename to tests/queries/0_stateless/02696_inverted_idx_checksums.sql diff --git a/tests/queries/0_stateless/25341_stop_reading_on_first_cancel.reference b/tests/queries/0_stateless/02697_stop_reading_on_first_cancel.reference similarity index 100% rename from tests/queries/0_stateless/25341_stop_reading_on_first_cancel.reference rename to tests/queries/0_stateless/02697_stop_reading_on_first_cancel.reference diff --git a/tests/queries/0_stateless/25341_stop_reading_on_first_cancel.sh b/tests/queries/0_stateless/02697_stop_reading_on_first_cancel.sh similarity index 100% rename from tests/queries/0_stateless/25341_stop_reading_on_first_cancel.sh rename to tests/queries/0_stateless/02697_stop_reading_on_first_cancel.sh diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.reference b/tests/queries/0_stateless/02698_marked_dropped_tables.reference similarity index 100% rename from tests/queries/0_stateless/25400_marked_dropped_tables.reference rename to tests/queries/0_stateless/02698_marked_dropped_tables.reference diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.sql b/tests/queries/0_stateless/02698_marked_dropped_tables.sql similarity index 100% rename from tests/queries/0_stateless/25400_marked_dropped_tables.sql rename to tests/queries/0_stateless/02698_marked_dropped_tables.sql diff --git a/tests/queries/0_stateless/25401_polygons_sym_difference_rollup.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference similarity index 100% rename from tests/queries/0_stateless/25401_polygons_sym_difference_rollup.reference rename to tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference diff --git a/tests/queries/0_stateless/25401_polygons_sym_difference_rollup.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql similarity index 100% rename from tests/queries/0_stateless/25401_polygons_sym_difference_rollup.sql rename to tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql diff --git a/tests/queries/0_stateless/25401_regexp_operator.reference b/tests/queries/0_stateless/02700_regexp_operator.reference similarity index 100% rename from tests/queries/0_stateless/25401_regexp_operator.reference rename to tests/queries/0_stateless/02700_regexp_operator.reference diff --git a/tests/queries/0_stateless/25401_regexp_operator.sql b/tests/queries/0_stateless/02700_regexp_operator.sql similarity index 100% rename from tests/queries/0_stateless/25401_regexp_operator.sql rename to tests/queries/0_stateless/02700_regexp_operator.sql From 86f682855687cda2494acf33fa9cc63183da0883 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 15:56:31 +0000 Subject: [PATCH 52/73] Docs: Update syntax of some SHOW queries --- docs/en/sql-reference/statements/show.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 7d5e2845c4a..544c556d4b3 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -21,13 +21,13 @@ Note that if you use this statement to get `CREATE` query of system tables, you Prints a list of all databases. ```sql -SHOW DATABASES [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] +SHOW DATABASES [[NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] ``` This statement is identical to the query: ```sql -SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] +SELECT name FROM system.databases [WHERE name [NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] ``` ### Examples @@ -117,7 +117,7 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" Displays a list of tables. ```sql -SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [[NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` If the `FROM` clause is not specified, the query returns the list of tables from the current database. @@ -125,7 +125,7 @@ If the `FROM` clause is not specified, the query returns the list of tables from This statement is identical to the query: ```sql -SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SELECT name FROM system.tables [WHERE name [NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` ### Examples @@ -370,7 +370,7 @@ Returns a list of clusters. All available clusters are listed in the [system.clu ``` sql SHOW CLUSTER '' -SHOW CLUSTERS [LIKE|NOT LIKE ''] [LIMIT ] +SHOW CLUSTERS [[NOT] LIKE|ILIKE ''] [LIMIT ] ``` ### Examples From c39d931ef7db67a76aff3787f402b9ad42105010 Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Fri, 24 Mar 2023 09:07:33 +0100 Subject: [PATCH 53/73] Fix wide_integer_impl.h compilation in macOS M1 --- base/base/wide_integer_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 30d08be2e4f..712c8516dad 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -1242,7 +1242,7 @@ constexpr integer::operator long double() const noexcept for (unsigned i = 0; i < _impl::item_count; ++i) { long double t = res; - res *= std::numeric_limits::max(); + res *= std::numeric_limits::max(); res += t; res += tmp.items[_impl::big(i)]; } From 20fa73042d1f351a7aca1d7545fd16525d4fa5cb Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Fri, 24 Mar 2023 11:26:35 +0100 Subject: [PATCH 54/73] Fix AggregateFunctionQuantile.h compilation in macOS M1 --- src/AggregateFunctions/AggregateFunctionQuantile.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 49157acf690..21799ccd65a 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -125,7 +125,7 @@ public: if constexpr (std::is_same_v>) { /// QuantileTiming only supports unsigned integers. Too large values are also meaningless. - if (isNaN(value) || value > std::numeric_limits::max() || value < 0) + if (isNaN(value) || value > std::numeric_limits::max() || value < 0) return; } From 4eafb1ae6b03c20624da97260ad8af17cfe30841 Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Fri, 24 Mar 2023 23:58:59 +0100 Subject: [PATCH 55/73] Fix tests --- base/base/wide_integer_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 712c8516dad..7cdb527f9cf 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -1242,7 +1242,7 @@ constexpr integer::operator long double() const noexcept for (unsigned i = 0; i < _impl::item_count; ++i) { long double t = res; - res *= std::numeric_limits::max(); + res *= static_cast(std::numeric_limits::max()); res += t; res += tmp.items[_impl::big(i)]; } From b64cb10d205d0d0b8339dfba4beedca5359fc786 Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Mon, 27 Mar 2023 13:23:12 +0200 Subject: [PATCH 56/73] Rework AggregateFunctionQuantile.h fix --- src/AggregateFunctions/AggregateFunctionQuantile.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 21799ccd65a..01ff019f221 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -125,8 +125,15 @@ public: if constexpr (std::is_same_v>) { /// QuantileTiming only supports unsigned integers. Too large values are also meaningless. - if (isNaN(value) || value > std::numeric_limits::max() || value < 0) + #ifdef OS_DARWIN + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion" + #endif + if (isNaN(value) || value > std::numeric_limits::max() || value < 0) return; + #ifdef OS_DARWIN + #pragma clang diagnostic pop + #endif } if constexpr (has_second_arg) From feac46eac4b06ff21addd4985559dc7b53e28c2e Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Mon, 27 Mar 2023 11:16:28 -0600 Subject: [PATCH 57/73] Update dropped_tables.md (#48067) * Update dropped_tables.md * Update dropped_tables.md * Update docs/en/operations/system-tables/dropped_tables.md --------- Co-authored-by: Alexander Tokmakov --- docs/en/operations/system-tables/dropped_tables.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/system-tables/dropped_tables.md b/docs/en/operations/system-tables/dropped_tables.md index a260aea1c5e..cb6cec0035a 100644 --- a/docs/en/operations/system-tables/dropped_tables.md +++ b/docs/en/operations/system-tables/dropped_tables.md @@ -1,7 +1,7 @@ --- -slug: /en/operations/system-tables/marked_dropped_tables +slug: /en/operations/system-tables/dropped_tables --- -# marked_dropped_tables +# dropped_tables Contains information about tables that drop table has been executed but data cleanup has not been actually performed. @@ -17,11 +17,11 @@ Columns: **Example** -The following example shows how to get information about marked_dropped_tables. +The following example shows how to get information about dropped_tables. ``` sql SELECT * -FROM system.marked_dropped_tables\G +FROM system.dropped_tables\G ``` ``` text @@ -34,4 +34,4 @@ uuid: 03141bb2-e97a-4d7c-a172-95cc066bb3bd engine: MergeTree metadata_dropped_path: /data/ClickHouse/build/programs/data/metadata_dropped/default.test.03141bb2-e97a-4d7c-a172-95cc066bb3bd.sql table_dropped_time: 2023-03-16 23:43:31 -``` \ No newline at end of file +``` From 314ee1244229ee6adfb7340ea29158b5fbbd8b6e Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Mar 2023 16:54:40 +0200 Subject: [PATCH 58/73] Fix checks --- src/Storages/MergeTree/MergeTreeData.cpp | 24 ++++++++----- .../test_encrypted_disk/configs/storage.xml | 30 ++++++++++++++++ tests/integration/test_encrypted_disk/test.py | 35 ++++++++++--------- 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f7814fcf276..4d04b8ba69b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1521,21 +1521,27 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (const auto & disk_ptr : disks) { defined_disk_names.insert(disk_ptr->getName()); + } + /// In case of delegate disks it is not enough to traverse `disks`, + /// because for example cache or encrypted disk which wrap s3 disk and s3 disk itself can be put into different storage policies. + /// But disk->exists returns the same thing for both disks. + for (const auto & [disk_name, disk] : getContext()->getDisksMap()) + { /// As encrypted disk can use the same path of its nested disk, /// we need to take it into account here. - const auto & delegate = disk_ptr->getDelegateDiskIfExists(); - if (delegate && disk_ptr->getPath() == delegate->getPath()) + const auto & delegate = disk->getDelegateDiskIfExists(); + if (delegate && disk->getPath() == delegate->getPath()) defined_disk_names.insert(delegate->getName()); - /// As cache is implemented on object storage layer, not on disk level, e.g. - /// we have such structure: - /// DiskObjectStorage(CachedObjectStorage(...(CachedObjectStored(ObjectStorage)...))) - /// and disk_ptr->getName() here is the name of last delegate - ObjectStorage. - /// So now we need to add cache layers to defined disk names. - if (disk_ptr->supportsCache()) + if (disk->supportsCache()) { - auto caches = disk_ptr->getCacheLayersNames(); + /// As cache is implemented on object storage layer, not on disk level, e.g. + /// we have such structure: + /// DiskObjectStorage(CachedObjectStorage(...(CachedObjectStored(ObjectStorage)...))) + /// and disk_ptr->getName() here is the name of last delegate - ObjectStorage. + /// So now we need to add cache layers to defined disk names. + auto caches = disk->getCacheLayersNames(); defined_disk_names.insert(caches.begin(), caches.end()); } } diff --git a/tests/integration/test_encrypted_disk/configs/storage.xml b/tests/integration/test_encrypted_disk/configs/storage.xml index a6fff813f2e..1e48c80d50f 100644 --- a/tests/integration/test_encrypted_disk/configs/storage.xml +++ b/tests/integration/test_encrypted_disk/configs/storage.xml @@ -15,7 +15,23 @@ encrypted disk_s3 1234567812345678 + encrypted/ + + encrypted + disk_s3 + 1234567812345678 + + + disk_s3 + s3_cache/ + 1Gi + + + encrypted + s3_cache + 1234567812345678 + encrypted disk_local @@ -73,6 +89,20 @@ + + +
+ disk_s3_encrypted_default_path +
+ + _ + + +
+ encrypted_s3_cache +
+ + diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 8e2935a1603..681df89dd0f 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -273,24 +273,25 @@ def test_read_in_order(): def test_restart(): - node.query( - """ - DROP TABLE IF EXISTS encrypted_test; - CREATE TABLE encrypted_test ( - id Int64, - data String - ) ENGINE=MergeTree() - ORDER BY id - SETTINGS disk='disk_s3_encrypted' - """ - ) + for policy in ["disk_s3_encrypted_default_path", "encrypted_s3_cache"]: + node.query( + f""" + DROP TABLE IF EXISTS encrypted_test; + CREATE TABLE encrypted_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS disk='{policy}' + """ + ) - node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") - select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" - assert node.query(select_query) == "(0,'data'),(1,'data')" + node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") + select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" + assert node.query(select_query) == "(0,'data'),(1,'data')" - node.restart_clickhouse() + node.restart_clickhouse() - assert node.query(select_query) == "(0,'data'),(1,'data')" + assert node.query(select_query) == "(0,'data'),(1,'data')" - node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY;") + node.query("DROP TABLE encrypted_test NO DELAY;") From 98488f30ab25062e789acf40b6a8d13cc6b1e0cb Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 27 Mar 2023 14:40:45 -0300 Subject: [PATCH 59/73] Update settings.md --- docs/en/operations/settings/settings.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f3c0f20f3a6..eb94836b3ba 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -988,6 +988,17 @@ The interval in microseconds for checking whether request execution has been can Default value: 100,000 (checks for cancelling and sends the progress ten times per second). +## idle_connection_timeout {#idle_connection_timeout} + +Timeout to close idle TCP connections after specified number of seconds. + +Possible values: + +- Positive integer. +- 0 — Never. + +Default value: 3600. + ## connect_timeout, receive_timeout, send_timeout {#connect-timeout-receive-timeout-send-timeout} Timeouts in seconds on the socket used for communicating with the client. From 295d3b751013f0f946e7d189d9054bfdef6d3ad4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 27 Mar 2023 14:45:43 -0300 Subject: [PATCH 60/73] Update settings.md --- docs/en/operations/settings/settings.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index eb94836b3ba..4e5ae6d63cf 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -994,8 +994,7 @@ Timeout to close idle TCP connections after specified number of seconds. Possible values: -- Positive integer. -- 0 — Never. +- Positive integer (0 - close immediatly, after 0 seconds). Default value: 3600. From c296c2e299c1dd31b9bdf49a8f5d1125f7b86f97 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov Date: Mon, 27 Mar 2023 18:34:12 +0000 Subject: [PATCH 61/73] Review changes --- src/Core/Settings.h | 1 - src/Interpreters/InterpreterCreateQuery.cpp | 97 ++++++++----------- src/Interpreters/InterpreterCreateQuery.h | 2 - ...6_ignore_inacc_tables_mat_view_atttach.sql | 6 -- 4 files changed, 38 insertions(+), 68 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6670920f613..accfe387137 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -467,7 +467,6 @@ class IColumn; M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ - M(Bool, skip_materialized_view_checking_if_source_table_not_exist, false, "Allow attaching to a materialized view even if dependent tables are inaccessible.", 0) \ \ M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 4e07a6ab017..b142e4c3450 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -97,7 +97,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int UNKNOWN_DATABASE; - extern const int UNKNOWN_TABLE; extern const int PATH_ACCESS_DENIED; extern const int NOT_IMPLEMENTED; extern const int ENGINE_REQUIRED; @@ -1077,62 +1076,6 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data } } -void InterpreterCreateQuery::checkTypecompatibleForMaterializeView(const ASTCreateQuery & create) -{ - if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( - {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid}, - getContext() - )) - { - Block input_block; - try - { - if (getContext()->getSettingsRef().allow_experimental_analyzer) - { - input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); - } - else - { - input_block = InterpreterSelectWithUnionQuery(create.select->clone(), - getContext(), - SelectQueryOptions().analyze()).getSampleBlock(); - } - } - catch (const Exception & e) - { - if (getContext()->getSettingsRef().skip_materialized_view_checking_if_source_table_not_exist && - e.code() == ErrorCodes::UNKNOWN_TABLE && create.attach - ) - { - LOG_WARNING(&Poco::Logger::get("InterpreterSelectQuery"), "{}", e.message()); - return; - } - else - { - throw; - } - } - - Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); - - ColumnsWithTypeAndName input_columns; - ColumnsWithTypeAndName output_columns; - for (const auto & input_column : input_block) - { - if (const auto * output_column = output_block.findByName(input_column.name)) - { - input_columns.push_back(input_column.cloneEmpty()); - output_columns.push_back(output_column->cloneEmpty()); - } - } - - ActionsDAG::makeConvertingActions( - input_columns, - output_columns, - ActionsDAG::MatchColumnsMode::Position - ); - } -} BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { @@ -1258,9 +1201,45 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create); /// Check type compatible for materialized dest table and select columns - if (create.select && create.is_materialized_view && create.to_table_id) + if (create.select && create.is_materialized_view && create.to_table_id && !create.attach) { - checkTypecompatibleForMaterializeView(create); + if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( + {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid}, + getContext() + )) + { + Block input_block; + + if (getContext()->getSettingsRef().allow_experimental_analyzer) + { + input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); + } + else + { + input_block = InterpreterSelectWithUnionQuery(create.select->clone(), + getContext(), + SelectQueryOptions().analyze()).getSampleBlock(); + } + + Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock(); + + ColumnsWithTypeAndName input_columns; + ColumnsWithTypeAndName output_columns; + for (const auto & input_column : input_block) + { + if (const auto * output_column = output_block.findByName(input_column.name)) + { + input_columns.push_back(input_column.cloneEmpty()); + output_columns.push_back(output_column->cloneEmpty()); + } + } + + ActionsDAG::makeConvertingActions( + input_columns, + output_columns, + ActionsDAG::MatchColumnsMode::Position + ); + } } DatabasePtr database; diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 22e71715bcf..4d11387f44c 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -95,8 +95,6 @@ private: BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties); /// Inserts data in created table if it's CREATE ... SELECT BlockIO fillTableIfNeeded(const ASTCreateQuery & create); - /// Check type compatible for materialized dest table and select columns - void checkTypecompatibleForMaterializeView(const ASTCreateQuery & create); void assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const; diff --git a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql index 615737eacf0..25e0ddf2ef6 100644 --- a/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql +++ b/tests/queries/0_stateless/02696_ignore_inacc_tables_mat_view_atttach.sql @@ -10,13 +10,7 @@ DROP TABLE test_table; DETACH TABLE mview; -/* Check that we get an exception with the option. */ - -SET skip_materialized_view_checking_if_source_table_not_exist = 0; -ATTACH TABLE mview; --{serverError 60} - /* Check that we don't get an exception with the option. */ -SET skip_materialized_view_checking_if_source_table_not_exist = 1; ATTACH TABLE mview; /* Check if the data in the materialized view is updated after the restore.*/ From 6d8a2bbd48d7d753e52b9cfcc75b03bdb9b63dae Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Mon, 27 Mar 2023 14:54:05 -0400 Subject: [PATCH 62/73] standardize admonitions --- .../database-engines/materialized-mysql.md | 4 +-- .../materialized-postgresql.md | 2 +- docs/en/engines/database-engines/mysql.md | 2 +- .../table-engines/integrations/hdfs.md | 4 +-- .../table-engines/integrations/kafka.md | 2 +- .../integrations/materialized-postgresql.md | 2 +- .../table-engines/integrations/postgresql.md | 2 +- .../engines/table-engines/integrations/s3.md | 4 +-- .../mergetree-family/aggregatingmergetree.md | 2 +- .../mergetree-family/collapsingmergetree.md | 2 +- .../custom-partitioning-key.md | 4 +-- .../mergetree-family/graphitemergetree.md | 6 ++-- .../mergetree-family/invertedindexes.md | 2 +- .../mergetree-family/mergetree.md | 4 +-- .../mergetree-family/replacingmergetree.md | 4 +-- .../mergetree-family/replication.md | 2 +- .../mergetree-family/summingmergetree.md | 2 +- .../versionedcollapsingmergetree.md | 2 +- .../engines/table-engines/special/buffer.md | 2 +- docs/en/engines/table-engines/special/file.md | 2 +- .../example-datasets/star-schema.md | 2 +- docs/en/interfaces/formats.md | 28 +++++++++---------- docs/en/interfaces/http.md | 4 +-- docs/en/interfaces/postgresql.md | 2 +- .../third-party/client-libraries.md | 2 +- .../en/interfaces/third-party/integrations.md | 2 +- .../external-authenticators/kerberos.md | 6 ++-- docs/en/operations/opentelemetry.md | 2 +- docs/en/operations/query-cache.md | 2 +- .../settings.md | 4 +-- .../settings/merge-tree-settings.md | 2 +- .../operations/settings/settings-formats.md | 2 +- docs/en/operations/settings/settings-users.md | 2 +- docs/en/operations/settings/settings.md | 20 ++++++------- docs/en/operations/storing-data.md | 2 +- docs/en/operations/system-tables/parts.md | 2 +- .../operations/utilities/clickhouse-copier.md | 2 +- .../parametric-functions.md | 4 +-- docs/en/sql-reference/data-types/float.md | 2 +- docs/en/sql-reference/data-types/json.md | 4 +-- .../data-types/special-data-types/interval.md | 2 +- docs/en/sql-reference/dictionaries/index.md | 6 ++-- docs/en/sql-reference/distributed-ddl.md | 4 +-- .../sql-reference/functions/hash-functions.md | 4 +-- .../sql-reference/functions/introspection.md | 2 +- .../sql-reference/functions/nlp-functions.md | 2 +- .../functions/other-functions.md | 8 +++--- docs/en/sql-reference/operators/exists.md | 2 +- docs/en/sql-reference/operators/index.md | 2 +- .../sql-reference/statements/alter/column.md | 2 +- .../statements/alter/constraint.md | 2 +- .../statements/create/row-policy.md | 2 +- .../sql-reference/statements/create/table.md | 8 +++--- .../sql-reference/statements/create/user.md | 6 ++-- docs/en/sql-reference/statements/optimize.md | 2 +- docs/en/sql-reference/statements/system.md | 2 +- docs/en/sql-reference/statements/watch.md | 4 +-- docs/en/sql-reference/table-functions/file.md | 2 +- docs/en/sql-reference/table-functions/hdfs.md | 2 +- .../table-functions/hdfsCluster.md | 2 +- .../sql-reference/table-functions/iceberg.md | 2 +- .../en/sql-reference/table-functions/index.md | 2 +- docs/en/sql-reference/table-functions/s3.md | 2 +- .../table-functions/s3Cluster.md | 2 +- 64 files changed, 113 insertions(+), 113 deletions(-) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index 899c8d024f1..45719b1340e 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -6,7 +6,7 @@ sidebar_position: 70 # [experimental] MaterializedMySQL -:::warning +:::note This is an experimental feature that should not be used in production. ::: @@ -245,7 +245,7 @@ extra care needs to be taken. You may specify overrides for tables that do not exist yet. -:::warning +:::important It is easy to break replication with table overrides if not used with care. For example: * If an ALIAS column is added with a table override, and a column with the same name is later added to the source diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index b43f71a7576..32e3435afa2 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -145,7 +145,7 @@ FROM pg_class WHERE oid = 'postgres_table'::regclass; ``` -:::warning +:::note Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used. ::: diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index e4ff734d55f..7c8c3459ec5 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -60,7 +60,7 @@ These variables are supported: - `version` - `max_allowed_packet` -:::warning +:::note By now these variables are stubs and don't correspond to anything. ::: diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index 7c04a6594a6..1497ea47eca 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -64,7 +64,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2 - Indexes. - [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is possible, but not recommended. - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: @@ -110,7 +110,7 @@ Table consists of all the files in both directories (all files should satisfy fo CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV') ``` -:::warning +:::note If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 255ba06f056..e2a7304dc59 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -102,7 +102,7 @@ Examples: Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/integrations/materialized-postgresql.md b/docs/en/engines/table-engines/integrations/materialized-postgresql.md index 11e7928c3ed..3920b402a49 100644 --- a/docs/en/engines/table-engines/integrations/materialized-postgresql.md +++ b/docs/en/engines/table-engines/integrations/materialized-postgresql.md @@ -52,6 +52,6 @@ PRIMARY KEY key; SELECT key, value, _version FROM postgresql_db.postgresql_replica; ``` -:::warning +:::note Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used. ::: diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 18e884f3bcc..2222d1fc016 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -74,7 +74,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp PostgreSQL `Array` types are converted into ClickHouse arrays. -:::warning +:::note Be careful - in PostgreSQL an array data, created like a `type_name[]`, may contain multi-dimensional arrays of different dimensions in different table rows in same column. But in ClickHouse it is only allowed to have multidimensional arrays of the same count of dimensions in all table rows in same column. ::: diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index dd843945e10..2901edb9f35 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -63,7 +63,7 @@ For more information about virtual columns see [here](../../../engines/table-eng - Indexes. - [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is possible, but not supported. - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: @@ -78,7 +78,7 @@ For more information about virtual columns see [here](../../../engines/table-eng Constructions with `{}` are similar to the [remote](../../../sql-reference/table-functions/remote.md) table function. -:::warning +:::note If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md index 9677f75a358..6591f666244 100644 --- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -43,7 +43,7 @@ When creating an `AggregatingMergeTree` table the same [clauses](../../../engine Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch the old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md index 0bd665116f0..7e16f4926db 100644 --- a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -45,7 +45,7 @@ When creating a `CollapsingMergeTree` table, the same [query clauses](../../../e Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index b1e79c4c3fd..f5b8436fdfe 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -6,7 +6,7 @@ sidebar_label: Custom Partitioning Key # Custom Partitioning Key -:::warning +:::note In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. You should never use too granular of partitioning. Don't partition your data by client identifiers or names. Instead, make a client identifier or name the first column in the ORDER BY expression. @@ -159,7 +159,7 @@ FROM session_log GROUP BY UserID; ``` -:::warning +:::note Performance of such a query heavily depends on the table layout. Because of that the optimisation is not enabled by default. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 104ec049ec4..9577c8dc936 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -55,7 +55,7 @@ When creating a `GraphiteMergeTree` table, the same [clauses](../../../engines/t Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: @@ -129,7 +129,7 @@ default ... ``` -:::warning +:::important Patterns must be strictly ordered: 1. Patterns without `function` or `retention`. @@ -263,6 +263,6 @@ Valid values: ``` -:::warning +:::note Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). ::: diff --git a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md index aa11258dc4a..701615495de 100644 --- a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md @@ -15,7 +15,7 @@ tokenized cells of the string column. For example, the string cell "I will be a " wi", "wil", "ill", "ll ", "l b", " be" etc. The more fine-granular the input strings are tokenized, the bigger but also the more useful the resulting inverted index will be. -:::warning +:::note Inverted indexes are experimental and should not be used in production environments yet. They may change in the future in backward-incompatible ways, for example with respect to their DDL/DQL syntax or performance/compression characteristics. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index f1b7a40094d..fa97579c33e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -192,7 +192,7 @@ The `index_granularity` setting can be omitted because 8192 is the default value Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: @@ -1092,7 +1092,7 @@ Other parameters: Examples of working configurations can be found in integration tests directory (see e.g. [test_merge_tree_azure_blob_storage](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml) or [test_azure_blob_storage_zero_copy_replication](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_azure_blob_storage_zero_copy_replication/configs/config.d/storage_conf.xml)). - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index f5d81182898..8351a31db55 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -30,7 +30,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of request parameters, see [statement description](../../../sql-reference/statements/create/table.md). -:::warning +:::note Uniqueness of rows is determined by the `ORDER BY` table section, not `PRIMARY KEY`. ::: @@ -96,7 +96,7 @@ When creating a `ReplacingMergeTree` table the same [clauses](../../../engines/t Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index c50433f2aeb..b2b967c685a 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -43,7 +43,7 @@ ClickHouse uses [ClickHouse Keeper](/docs/en/guides/sre/keeper/index.md) for sto To use replication, set parameters in the [zookeeper](/docs/en/operations/server-configuration-parameters/settings.md/#server-settings_zookeeper) server configuration section. -:::warning +:::note Don’t neglect the security setting. ClickHouse supports the `digest` [ACL scheme](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) of the ZooKeeper security subsystem. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index b2b6272c58e..d0078656b5d 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -44,7 +44,7 @@ When creating a `SummingMergeTree` table the same [clauses](../../../engines/tab Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch the old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 2891907f79a..74ac9c97fc0 100644 --- a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -58,7 +58,7 @@ When creating a `VersionedCollapsingMergeTree` table, the same [clauses](../../. Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 5f28fafbc26..f7d84b9b452 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -86,7 +86,7 @@ If the set of columns in the Buffer table does not match the set of columns in a If the types do not match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log, and the buffer is cleared. The same happens if the subordinate table does not exist when the buffer is flushed. -:::warning +:::note Running ALTER on the Buffer table in releases made before 26 Oct 2021 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117) and [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565)), so deleting the Buffer table and then recreating is the only option. Check that this error is fixed in your release before trying to run ALTER on the Buffer table. ::: diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md index 8314c511236..e99b1c83cbc 100644 --- a/docs/en/engines/table-engines/special/file.md +++ b/docs/en/engines/table-engines/special/file.md @@ -31,7 +31,7 @@ When creating table using `File(Format)` it creates empty subdirectory in that f You may manually create this subfolder and file in server filesystem and then [ATTACH](../../../sql-reference/statements/attach.md) it to table information with matching name, so you can query data from that file. -:::warning +:::note Be careful with this functionality, because ClickHouse does not keep track of external changes to such files. The result of simultaneous writes via ClickHouse and outside of ClickHouse is undefined. ::: diff --git a/docs/en/getting-started/example-datasets/star-schema.md b/docs/en/getting-started/example-datasets/star-schema.md index 1702be70410..72ced87ef55 100644 --- a/docs/en/getting-started/example-datasets/star-schema.md +++ b/docs/en/getting-started/example-datasets/star-schema.md @@ -18,7 +18,7 @@ $ make Generating data: -:::warning +:::note With `-s 100` dbgen generates 600 million rows (67 GB), while while `-s 1000` it generates 6 billion rows (which takes a lot of time) ::: diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ae3756d5d41..8430946a6c6 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -205,7 +205,7 @@ Differs from the `TabSeparated` format in that the column names are written in t During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness. -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from the input data will be mapped to the columns of the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -217,7 +217,7 @@ This format is also available under the name `TSVWithNames`. Differs from the `TabSeparated` format in that the column names are written to the first row, while the column types are in the second row. -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from the input data will be mapped to the columns in the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -470,7 +470,7 @@ The CSV format supports the output of totals and extremes the same way as `TabSe Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -480,7 +480,7 @@ Otherwise, the first row will be skipped. Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -500,7 +500,7 @@ There is also `CustomSeparatedIgnoreSpaces` format, which is similar to [Templat Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -510,7 +510,7 @@ Otherwise, the first row will be skipped. Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -969,7 +969,7 @@ Differs from `JSONEachRow`/`JSONStringsEachRow` in that ClickHouse will also yie Differs from `JSONCompactEachRow` format in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -979,7 +979,7 @@ Otherwise, the first row will be skipped. Differs from `JSONCompactEachRow` format in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -991,7 +991,7 @@ the types from input data will be compared with the types of the corresponding c Differs from `JSONCompactStringsEachRow` in that in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1001,7 +1001,7 @@ Otherwise, the first row will be skipped. Differs from `JSONCompactStringsEachRow` in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1120,7 +1120,7 @@ CREATE TABLE IF NOT EXISTS example_table - If `input_format_defaults_for_omitted_fields = 0`, then the default value for `x` and `a` equals `0` (as the default value for the `UInt32` data type). - If `input_format_defaults_for_omitted_fields = 1`, then the default value for `x` equals `0`, but the default value of `a` equals `x * 2`. -:::warning +:::note When inserting data with `input_format_defaults_for_omitted_fields = 1`, ClickHouse consumes more computational resources, compared to insertion with `input_format_defaults_for_omitted_fields = 0`. ::: @@ -1450,7 +1450,7 @@ Similar to [RowBinary](#rowbinary), but with added header: - [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N) - N `String`s specifying column names -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1464,7 +1464,7 @@ Similar to [RowBinary](#rowbinary), but with added header: - N `String`s specifying column names - N `String`s specifying column types -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1915,7 +1915,7 @@ SET format_avro_schema_registry_url = 'http://schema-registry'; SELECT * FROM topic1_stream; ``` -:::warning +:::note Setting `format_avro_schema_registry_url` needs to be configured in `users.xml` to maintain it’s value after a restart. Also you can use the `format_avro_schema_registry_url` setting of the `Kafka` table engine. ::: diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 32f323a63d5..4bc108cac7c 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -446,7 +446,7 @@ Next are the configuration methods for different `type`. The following example defines the values of [max_threads](../operations/settings/settings.md#settings-max_threads) and `max_final_threads` settings, then queries the system table to check whether these settings were set successfully. -:::warning +:::note To keep the default `handlers` such as` query`, `play`,` ping`, add the `` rule. ::: @@ -477,7 +477,7 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_final_threads 2 ``` -:::warning +:::note In one `predefined_query_handler` only supports one `query` of an insert type. ::: diff --git a/docs/en/interfaces/postgresql.md b/docs/en/interfaces/postgresql.md index f7a619ca620..1146274b012 100644 --- a/docs/en/interfaces/postgresql.md +++ b/docs/en/interfaces/postgresql.md @@ -54,7 +54,7 @@ default=> And that's it! You now have a PostgreSQL client connected to ClickHouse, and all commands and queries are executed on ClickHouse. -:::caution +:::note The PostgreSQL protocol currently only supports plain-text passwords. ::: diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index 0e065cb7179..1069a04391f 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -6,7 +6,7 @@ sidebar_label: Client Libraries # Client Libraries from Third-party Developers -:::warning +:::note ClickHouse Inc does **not** maintain the libraries listed below and hasn’t done any extensive testing to ensure their quality. ::: diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 90a4f088be7..a089b3eff17 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -6,7 +6,7 @@ sidebar_label: Integrations # Integration Libraries from Third-party Developers -:::warning Disclaimer +:::note Disclaimer ClickHouse, Inc. does **not** maintain the tools and libraries listed below and haven’t done extensive testing to ensure their quality. ::: diff --git a/docs/en/operations/external-authenticators/kerberos.md b/docs/en/operations/external-authenticators/kerberos.md index b7a11d7445b..27dc23c9792 100644 --- a/docs/en/operations/external-authenticators/kerberos.md +++ b/docs/en/operations/external-authenticators/kerberos.md @@ -59,11 +59,11 @@ With filtering by realm: ``` -:::warning +:::note You can define only one `kerberos` section. The presence of multiple `kerberos` sections will force ClickHouse to disable Kerberos authentication. ::: -:::warning +:::note `principal` and `realm` sections cannot be specified at the same time. The presence of both `principal` and `realm` sections will force ClickHouse to disable Kerberos authentication. ::: @@ -103,7 +103,7 @@ Example (goes into `users.xml`): ``` -:::warning +:::note Note that Kerberos authentication cannot be used alongside with any other authentication mechanism. The presence of any other sections like `password` alongside `kerberos` will force ClickHouse to shutdown. ::: diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md index 1de5a09db0c..15185f7ae6b 100644 --- a/docs/en/operations/opentelemetry.md +++ b/docs/en/operations/opentelemetry.md @@ -7,7 +7,7 @@ title: "[experimental] Tracing ClickHouse with OpenTelemetry" [OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry. -:::warning +:::note This is an experimental feature that will change in backwards-incompatible ways in future releases. ::: diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index a5afee63e6e..ff4236ab8c3 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -29,7 +29,7 @@ Transactionally inconsistent caching is traditionally provided by client tools o the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance effort and avoids redundancy. -:::warning +:::note The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query processing) where wrong results are returned. ::: diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index bd0fb03bad0..c2c577d3d79 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -25,7 +25,7 @@ Default value: 3600. Data compression settings for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-engine tables. -:::warning +:::note Don’t use it if you have just started using ClickHouse. ::: @@ -1368,7 +1368,7 @@ The following settings are available: Changed settings take effect immediately. -:::warning +:::note Data for the query cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `max_size` or disable the query cache altogether. ::: diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 5bc174727ad..7e77394729a 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -289,7 +289,7 @@ Default value: 0 (seconds) When this setting has a value greater than than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. -:::warning Zero-copy replication is not ready for production +:::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index aa991cd9f15..40ca914b738 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -142,7 +142,7 @@ y Nullable(String) z IPv4 ``` -:::warning +:::note If the `schema_inference_hints` is not formated properly, or if there is a typo or a wrong datatype, etc... the whole schema_inference_hints will be ignored. ::: diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index 9b27af61851..bafac853377 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -118,7 +118,7 @@ To open access for user from any network, specify: ::/0 ``` -:::warning +:::note It’s insecure to open access from any network unless you have a firewall properly configured or the server is not directly connected to Internet. ::: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f3c0f20f3a6..239931b6757 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -460,7 +460,7 @@ Possible values: Changes the behaviour of join operations with `ANY` strictness. -:::warning +:::note This setting applies only for `JOIN` operations with [Join](../../engines/table-engines/special/join.md) engine tables. ::: @@ -550,7 +550,7 @@ Default value: 64. Enables legacy ClickHouse server behaviour in `ANY INNER|LEFT JOIN` operations. -:::warning +:::note Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour. ::: @@ -942,7 +942,7 @@ Higher values will lead to higher memory usage. The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying a smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. -:::warning +:::note This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. ::: @@ -960,7 +960,7 @@ We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won’t be decompressed. -:::warning +:::note This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. ::: @@ -1247,7 +1247,7 @@ Possible values: Default value: 1. -:::warning +:::note Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas) without [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key). If [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) is set, disable this setting only if it's used on a cluster with multiple shards containing multiple replicas. If it's used on a cluster with a single shard and multiple replicas, disabling this setting will have negative effects. @@ -1277,7 +1277,7 @@ Default value: `1`. This options will produce different results depending on the settings used. -:::warning +:::note This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. ::: @@ -2186,7 +2186,7 @@ Default value: 0. This setting also affects broken batches (that may appears because of abnormal server (machine) termination and no `fsync_after_insert`/`fsync_directories` for [Distributed](../../engines/table-engines/special/distributed.md) table engine). ::: -:::warning +:::note You should not rely on automatic batch splitting, since this may hurt performance. ::: @@ -2194,7 +2194,7 @@ You should not rely on automatic batch splitting, since this may hurt performanc Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core. -:::warning +:::note To use this setting, you need to set the `CAP_SYS_NICE` capability. The `clickhouse-server` package sets it up during installation. Some virtual environments do not allow you to set the `CAP_SYS_NICE` capability. In this case, `clickhouse-server` shows a message about it at the start. ::: @@ -2858,11 +2858,11 @@ Possible values: Default value: `0`. -:::warning +:::note Nullable primary key usually indicates bad design. It is forbidden in almost all main stream DBMS. The feature is mainly for [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) and is not heavily tested. Use with care. ::: -:::warning +:::note Do not enable this feature in version `<= 21.8`. It's not properly implemented and may lead to server crash. ::: diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index e019a3741cf..af7c526e29b 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -471,6 +471,6 @@ Use [http_max_single_read_retries](/docs/en/operations/settings/settings.md/#htt Zero-copy replication is possible, but not recommended, with `S3` and `HDFS` disks. Zero-copy replication means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself. -:::warning Zero-copy replication is not ready for production +:::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/operations/system-tables/parts.md b/docs/en/operations/system-tables/parts.md index 106d3c59dea..e7700562e35 100644 --- a/docs/en/operations/system-tables/parts.md +++ b/docs/en/operations/system-tables/parts.md @@ -99,7 +99,7 @@ Columns: - `move_ttl_info.expression` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Array of expressions. Each expression defines a [TTL MOVE rule](../../engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-ttl). -:::warning +:::note The `move_ttl_info.expression` array is kept mostly for backward compatibility, now the simpliest way to check `TTL MOVE` rule is to use the `move_ttl_info.min` and `move_ttl_info.max` fields. ::: diff --git a/docs/en/operations/utilities/clickhouse-copier.md b/docs/en/operations/utilities/clickhouse-copier.md index 87280bc3ba8..ccce7ea1b79 100644 --- a/docs/en/operations/utilities/clickhouse-copier.md +++ b/docs/en/operations/utilities/clickhouse-copier.md @@ -8,7 +8,7 @@ sidebar_label: clickhouse-copier Copies data from the tables in one cluster to tables in another (or the same) cluster. -:::warning +:::note To get a consistent copy, the data in the source tables and partitions should not change during the entire process. ::: diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 40184c0aa02..3b874dbe7cd 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -90,7 +90,7 @@ Checks whether the sequence contains an event chain that matches the pattern. sequenceMatch(pattern)(timestamp, cond1, cond2, ...) ``` -:::warning +:::note Events that occur at the same second may lay in the sequence in an undefined order affecting the result. ::: @@ -176,7 +176,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM Counts the number of event chains that matched the pattern. The function searches event chains that do not overlap. It starts to search for the next chain after the current chain is matched. -:::warning +:::note Events that occur at the same second may lay in the sequence in an undefined order affecting the result. ::: diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md index 38c414fa8cd..c89b24ad235 100644 --- a/docs/en/sql-reference/data-types/float.md +++ b/docs/en/sql-reference/data-types/float.md @@ -6,7 +6,7 @@ sidebar_label: Float32, Float64 # Float32, Float64 -:::warning +:::note If you need accurate calculations, in particular if you work with financial or business data requiring a high precision you should consider using Decimal instead. Floats might lead to inaccurate results as illustrated below: ``` diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index a21898de9a2..f727f0d75f7 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -6,7 +6,7 @@ sidebar_label: JSON # JSON -:::warning +:::note This feature is experimental and is not production ready. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json.md) instead. ::: @@ -14,7 +14,7 @@ Stores JavaScript Object Notation (JSON) documents in a single column. `JSON` is an alias for `Object('json')`. -:::warning +:::note The JSON data type is an experimental feature. To use it, set `allow_experimental_object_type = 1`. ::: diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 5169bc646c9..0ac5248e36c 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -8,7 +8,7 @@ sidebar_label: Interval The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator. -:::warning +:::note `Interval` data type values can’t be stored in tables. ::: diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index 2185e2b31c1..f697b1ecdcf 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -417,7 +417,7 @@ Example: The table contains discounts for each advertiser in the format: To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](#dictionary-key-and-fields). These elements must contain elements `name` and `type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others). -:::warning +:::note Values of `range_min` and `range_max` should fit in `Int64` type. ::: @@ -706,7 +706,7 @@ Set a large enough cache size. You need to experiment to select the number of ce 3. Assess memory consumption using the `system.dictionaries` table. 4. Increase or decrease the number of cells until the required memory consumption is reached. -:::warning +:::note Do not use ClickHouse as a source, because it is slow to process queries with random reads. ::: @@ -1888,7 +1888,7 @@ ClickHouse supports the following types of keys: An xml structure can contain either `` or ``. DDL-query must contain single `PRIMARY KEY`. -:::warning +:::note You must not describe key as an attribute. ::: diff --git a/docs/en/sql-reference/distributed-ddl.md b/docs/en/sql-reference/distributed-ddl.md index ff5155391be..d170f3765c2 100644 --- a/docs/en/sql-reference/distributed-ddl.md +++ b/docs/en/sql-reference/distributed-ddl.md @@ -18,6 +18,6 @@ In order to run these queries correctly, each host must have the same cluster de The local version of the query will eventually be executed on each host in the cluster, even if some hosts are currently not available. -:::warning +:::important The order for executing queries within a single host is guaranteed. -::: \ No newline at end of file +::: diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 69dc73e2fb0..2943ba13861 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -125,7 +125,7 @@ SELECT sipHash64Keyed((506097522914230528, 1084818905618843912), array('e','x',' Like [sipHash64](#hash_functions-siphash64) but produces a 128-bit hash value, i.e. the final xor-folding state is done up to 128 bits. -:::warning +:::note This 128-bit variant differs from the reference implementation and it's weaker. This version exists because, when it was written, there was no official 128-bit extension for SipHash. New projects should probably use [sipHash128Reference](#hash_functions-siphash128reference). @@ -165,7 +165,7 @@ Result: Same as [sipHash128](#hash_functions-siphash128) but additionally takes an explicit key argument instead of using a fixed key. -:::warning +:::note This 128-bit variant differs from the reference implementation and it's weaker. This version exists because, when it was written, there was no official 128-bit extension for SipHash. New projects should probably use [sipHash128ReferenceKeyed](#hash_functions-siphash128referencekeyed). diff --git a/docs/en/sql-reference/functions/introspection.md b/docs/en/sql-reference/functions/introspection.md index 9357f75b8e6..c01abd6ed89 100644 --- a/docs/en/sql-reference/functions/introspection.md +++ b/docs/en/sql-reference/functions/introspection.md @@ -8,7 +8,7 @@ sidebar_label: Introspection You can use functions described in this chapter to introspect [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) and [DWARF](https://en.wikipedia.org/wiki/DWARF) for query profiling. -:::warning +:::note These functions are slow and may impose security considerations. ::: diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md index f68448af2be..132a126edee 100644 --- a/docs/en/sql-reference/functions/nlp-functions.md +++ b/docs/en/sql-reference/functions/nlp-functions.md @@ -5,7 +5,7 @@ sidebar_label: NLP title: "[experimental] Natural Language Processing functions" --- -:::warning +:::note This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in future releases. Set `allow_experimental_nlp_functions = 1` to enable it. ::: diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 011b73405c5..2e44fa5e9f6 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -792,7 +792,7 @@ neighbor(column, offset[, default_value]) The result of the function depends on the affected data blocks and the order of data in the block. -:::warning +:::tip It can reach the neighbor rows only inside the currently processed data block. ::: @@ -902,7 +902,7 @@ Result: Calculates the difference between successive row values ​​in the data block. Returns 0 for the first row and the difference from the previous row for each subsequent row. -:::warning +:::tip It can reach the previous row only inside the currently processed data block. ::: @@ -986,7 +986,7 @@ Each event has a start time and an end time. The start time is included in the e The function calculates the total number of active (concurrent) events for each event start time. -:::warning +:::tip Events must be ordered by the start time in ascending order. If this requirement is violated the function raises an exception. Every data block is processed separately. If events from different data blocks overlap then they can not be processed correctly. ::: @@ -1674,7 +1674,7 @@ Result: Accumulates states of an aggregate function for each row of a data block. -:::warning +:::tip The state is reset for each new data block. ::: diff --git a/docs/en/sql-reference/operators/exists.md b/docs/en/sql-reference/operators/exists.md index 4bc29389c9c..5e96e11b924 100644 --- a/docs/en/sql-reference/operators/exists.md +++ b/docs/en/sql-reference/operators/exists.md @@ -7,7 +7,7 @@ The `EXISTS` operator checks how many records are in the result of a subquery. I `EXISTS` can be used in a [WHERE](../../sql-reference/statements/select/where.md) clause. -:::warning +:::tip References to main query tables and columns are not supported in a subquery. ::: diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 0fe7ebbf4b6..ade2d601f80 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -229,7 +229,7 @@ Types of intervals: You can also use a string literal when setting the `INTERVAL` value. For example, `INTERVAL 1 HOUR` is identical to the `INTERVAL '1 hour'` or `INTERVAL '1' hour`. -:::warning +:::tip Intervals with different types can’t be combined. You can’t use expressions like `INTERVAL 4 DAY 1 HOUR`. Specify intervals in units that are smaller or equal to the smallest unit of the interval, for example, `INTERVAL 25 HOUR`. You can use consecutive operations, like in the example below. ::: diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index d580efa4992..921785102a8 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -75,7 +75,7 @@ Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly. -:::warning +:::tip You can’t delete a column if it is referenced by [materialized view](/docs/en/sql-reference/statements/create/view.md/#materialized). Otherwise, it returns an error. ::: diff --git a/docs/en/sql-reference/statements/alter/constraint.md b/docs/en/sql-reference/statements/alter/constraint.md index 844b24d7374..7a8f5809320 100644 --- a/docs/en/sql-reference/statements/alter/constraint.md +++ b/docs/en/sql-reference/statements/alter/constraint.md @@ -17,7 +17,7 @@ See more on [constraints](../../../sql-reference/statements/create/table.md#cons Queries will add or remove metadata about constraints from table so they are processed immediately. -:::warning +:::tip Constraint check **will not be executed** on existing data if it was added. ::: diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index 56a57534234..aa0a07747f2 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -7,7 +7,7 @@ title: "CREATE ROW POLICY" Creates a [row policy](../../../guides/sre/user-management/index.md#row-policy-management), i.e. a filter used to determine which rows a user can read from a table. -:::warning +:::tip Row policies makes sense only for users with readonly access. If user can modify table or copy partitions between tables, it defeats the restrictions of row policies. ::: diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index b29bf31e26c..3432066864f 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -285,7 +285,7 @@ ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); ``` -:::warning +:::tip You can't combine both ways in one query. ::: @@ -343,7 +343,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -:::warning +:::tip You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. ::: @@ -438,11 +438,11 @@ Encryption codecs: These codecs use a fixed nonce and encryption is therefore deterministic. This makes it compatible with deduplicating engines such as [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md) but has a weakness: when the same data block is encrypted twice, the resulting ciphertext will be exactly the same so an adversary who can read the disk can see this equivalence (although only the equivalence, without getting its content). -:::warning +:::note Most engines including the "\*MergeTree" family create index files on disk without applying codecs. This means plaintext will appear on disk if an encrypted column is indexed. ::: -:::warning +:::note If you perform a SELECT query mentioning a specific value in an encrypted column (such as in its WHERE clause), the value may appear in [system.query_log](../../../operations/system-tables/query_log.md). You may want to disable the logging. ::: diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index a38523ee37b..b9bd2c1c507 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -50,7 +50,7 @@ There are multiple ways of user identification: CREATE USER name2 IDENTIFIED WITH plaintext_password BY 'my_password' ``` - :::warning + :::tip The password is stored in a SQL text file in `/var/lib/clickhouse/access`, so it's not a good idea to use `plaintext_password`. Try `sha256_password` instead, as demonstrated next... ::: @@ -76,7 +76,7 @@ There are multiple ways of user identification: ATTACH USER name3 IDENTIFIED WITH sha256_hash BY '0C268556C1680BEF0640AAC1E7187566704208398DA31F03D18C74F5C5BE5053' SALT '4FB16307F5E10048196966DD7E6876AE53DE6A1D1F625488482C75F14A5097C7'; ``` - :::note + :::tip If you have already created a hash value and corresponding salt value for a username, then you can use `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'`. For identification with `sha256_hash` using `SALT` - hash must be calculated from concatenation of 'password' and 'salt'. ::: @@ -109,7 +109,7 @@ Another way of specifying host is to use `@` syntax following the username. Exam - `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` syntax. - `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` syntax. -:::warning +:::tip ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we do not recommend to do so. ::: diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 78615a2f9ad..427ee75cd5f 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -7,7 +7,7 @@ title: "OPTIMIZE Statement" This query tries to initialize an unscheduled merge of data parts for tables. -:::warning +:::note `OPTIMIZE` can’t fix the `Too many parts` error. ::: diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index d069ae8413a..b0877d94e8a 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -312,7 +312,7 @@ One may execute query after: Replica attaches locally found parts and sends info about them to Zookeeper. Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). -:::warning +:::note Parts in all states are moved to `detached/` folder. Parts active before data loss (committed) are attached. ::: diff --git a/docs/en/sql-reference/statements/watch.md b/docs/en/sql-reference/statements/watch.md index 90d19e6be0e..5230479cbd2 100644 --- a/docs/en/sql-reference/statements/watch.md +++ b/docs/en/sql-reference/statements/watch.md @@ -6,7 +6,7 @@ sidebar_label: WATCH # WATCH Statement (Experimental) -:::warning +:::note This is an experimental feature that may change in backwards-incompatible ways in the future releases. Enable live views and `WATCH` query using `set allow_experimental_live_view = 1`. ::: @@ -107,4 +107,4 @@ The `FORMAT` clause works the same way as for the [SELECT](../../sql-reference/s :::note The [JSONEachRowWithProgress](../../interfaces/formats.md#jsoneachrowwithprogress) format should be used when watching [LIVE VIEW](./create/view.md#live-view) tables over the HTTP interface. The progress messages will be added to the output to keep the long-lived HTTP connection alive until the query result changes. The interval between progress messages is controlled using the [live_view_heartbeat_interval](./create/view.md#live-view-settings) setting. -::: \ No newline at end of file +::: diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index 594c328c3ff..48c2cadc62c 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -109,7 +109,7 @@ Query the number of rows in all files of these two directories: SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32'); ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md index 97a253a5356..1b4588a9b55 100644 --- a/docs/en/sql-reference/table-functions/hdfs.md +++ b/docs/en/sql-reference/table-functions/hdfs.md @@ -79,7 +79,7 @@ SELECT count(*) FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/hdfsCluster.md b/docs/en/sql-reference/table-functions/hdfsCluster.md index 231c552610f..546c6a3d1b1 100644 --- a/docs/en/sql-reference/table-functions/hdfsCluster.md +++ b/docs/en/sql-reference/table-functions/hdfsCluster.md @@ -50,7 +50,7 @@ SELECT count(*) FROM hdfsCluster('cluster_simple', 'hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 7c2648d3dcf..713b0f9bbf5 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -32,7 +32,7 @@ A table with the specified structure for reading data in the specified Iceberg t SELECT * FROM iceberg('http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') ``` -:::warning +:::important ClickHouse currently supports reading v1 (v2 support is coming soon!) of the Iceberg format via the `iceberg` table function and `Iceberg` table engine. ::: diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index b49c2f8da20..1010d53e86d 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -20,6 +20,6 @@ You can use table functions in: - [INSERT INTO TABLE FUNCTION](../../sql-reference/statements/insert-into.md#inserting-into-table-function) query. -:::warning +:::note You can’t use table functions if the [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) setting is disabled. ::: diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 99b7832394d..44d1efdee7a 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -118,7 +118,7 @@ FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/ └─────────┘ ``` -:::warning +:::tip If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index f420a69596c..504f92b4dc0 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -42,7 +42,7 @@ SELECT * FROM s3Cluster( Count the total amount of rows in all files in the cluster `cluster_simple`: -:::warning +:::tip If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: From c7c9d8a92adc44636eb0ba9c5b041bcfa39a8fb7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 28 Mar 2023 01:05:10 +0300 Subject: [PATCH 63/73] Update README.md --- programs/obfuscator/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/obfuscator/README.md b/programs/obfuscator/README.md index 8c2aaafb3ac..371af6344ef 100644 --- a/programs/obfuscator/README.md +++ b/programs/obfuscator/README.md @@ -312,7 +312,7 @@ hash("images/c") % total_count: ^ 
 

-PhotoFunia - Haber7 - Hava mükemment.net Oynamak içinde şaşıracak haber, Oyunu Oynanılmaz • apród.hu kínálatában - RT Arabic
+PhotoFunia - Haber7 - Have mükemment.net Oynamak içinde şaşıracak haber, Oyunu Oynanılmaz • apród.hu kínálatában - RT Arabic
 PhotoFunia - Kinobar.Net - apród: Ingyenes | Posti
 PhotoFunia - Peg Perfeo - Castika, Sıradışı Deniz Lokoning Your Code, sire Eminema.tv/
 PhotoFunia - TUT.BY - Your Ayakkanın ve Son Dakika Spor,
@@ -323,7 +323,7 @@ PhotoFunia Monstelli'nin En İyi kisa.com.tr –Star Thunder Ekranı
 PhotoFunia Seks - Politika,Ekonomi,Spor GTA SANAYİ VE
 PhotoFunia Taker-Rating Star TV Resmi Söylenen Yatağa każdy dzież wierzchnie
 PhotoFunia TourIndex.Marketime oyunu Oyna Geldolları Mynet Spor,Magazin,Haberler yerel Haberleri ve Solvia, korkusuz Ev SahneTv
-PhotoFunia todo in the Gratis Perky Parti'nin yapıyı bu fotogram
+PhotoFunia todo in the Gratis Perky Parti'nin yapıyı by fotogram
 PhotoFunian Dünyasın takımız halles en kulları - TEZ
From 3946e66272caf05cb162e453f41b98318dfe7966 Mon Sep 17 00:00:00 2001 From: Jordi Villar Date: Tue, 28 Mar 2023 08:11:37 +0200 Subject: [PATCH 64/73] Comestic PR changes --- src/AggregateFunctions/AggregateFunctionQuantile.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 01ff019f221..85eaf6ceba6 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -125,15 +125,15 @@ public: if constexpr (std::is_same_v>) { /// QuantileTiming only supports unsigned integers. Too large values are also meaningless. - #ifdef OS_DARWIN - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion" - #endif +#ifdef OS_DARWIN +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion" +#endif if (isNaN(value) || value > std::numeric_limits::max() || value < 0) return; - #ifdef OS_DARWIN - #pragma clang diagnostic pop - #endif +#ifdef OS_DARWIN +# pragma clang diagnostic pop +#endif } if constexpr (has_second_arg) From 943a4f75f85006a9e6c0c07132318b30d992e1c2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 28 Mar 2023 12:00:56 +0200 Subject: [PATCH 65/73] Catch all exceptions to avoid replication stuck --- .../PostgreSQL/PostgreSQLReplicationHandler.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index f9bfe1d174a..322ad3c78c0 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -417,7 +417,15 @@ void PostgreSQLReplicationHandler::consumerFunc() { assertInitialized(); - bool schedule_now = getConsumer()->consume(); + bool schedule_now = true; + try + { + schedule_now = getConsumer()->consume(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } if (stop_synchronization) { From e5994e873e8aa8d44213e49956b64704b292daec Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 28 Mar 2023 14:03:50 +0200 Subject: [PATCH 66/73] Added documentation for new settings to disallow concurrent backup/restore --- docs/en/operations/backup.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index d58dd1376eb..46c3a52b5c7 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -383,3 +383,19 @@ Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ... For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions). A third-party tool is available to automate this approach: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup). + +## Settings to disallow concurrent backup/restore + +To disallow concurrent backup/restore, you can use these settings respectively. + +```xml + + + false + false + + +``` + +The default value for both is true, so by default concurrent backup/restores are allowed. +When this setting is used on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file From aace3cb34cd2aca833fb1faa2180dd4ed974b389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 28 Mar 2023 13:55:16 +0200 Subject: [PATCH 67/73] Try to prevent large test numbers --- utils/check-style/check-style | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 946e29856d1..7a1fa6ce123 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -165,7 +165,7 @@ find $ROOT_PATH/tests/queries -iname '*fail*' | grep . && echo 'Tests should not be named with "fail" in their names. It makes looking at the results less convenient when you search for "fail" substring in browser.' # Queries to system.query_log/system.query_thread_log should have current_database = currentDatabase() condition -# NOTE: it is not that accuate, but at least something. +# NOTE: it is not that accurate, but at least something. tests_with_query_log=( $( find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' | grep -vP $EXCLUDE_DIRS | @@ -177,6 +177,17 @@ for test_case in "${tests_with_query_log[@]}"; do } || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case" done +# There shouldn't be large jumps between test numbers (since they should be consecutive) +max_diff=$( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' | + grep -oP '\d+\D+\K\d+' | sort -n -u | awk 's{print ($0-s) " diff " s " and " $0 }{s=$0}' | sort -n | tail -n 1 +) +max_diff_value=( $(echo $max_diff) ) +if [[ $max_diff_value -ge 100 ]]; +then + echo "Too big of a difference between test numbers: $max_diff" +fi + # Queries to: tables_with_database_column=( system.tables From 07cc2a7cb25c560377437beb3e4854b4206ae93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 28 Mar 2023 14:06:24 +0200 Subject: [PATCH 68/73] Fix tests style --- ...id_having.reference => 02701_fail_on_invalid_having.reference} | 0 ...ail_on_invalid_having.sql => 02701_fail_on_invalid_having.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{25402_fail_on_invalid_having.reference => 02701_fail_on_invalid_having.reference} (100%) rename tests/queries/0_stateless/{25402_fail_on_invalid_having.sql => 02701_fail_on_invalid_having.sql} (100%) diff --git a/tests/queries/0_stateless/25402_fail_on_invalid_having.reference b/tests/queries/0_stateless/02701_fail_on_invalid_having.reference similarity index 100% rename from tests/queries/0_stateless/25402_fail_on_invalid_having.reference rename to tests/queries/0_stateless/02701_fail_on_invalid_having.reference diff --git a/tests/queries/0_stateless/25402_fail_on_invalid_having.sql b/tests/queries/0_stateless/02701_fail_on_invalid_having.sql similarity index 100% rename from tests/queries/0_stateless/25402_fail_on_invalid_having.sql rename to tests/queries/0_stateless/02701_fail_on_invalid_having.sql From 95a1799a006e5842be08138416d9f95234ec6df3 Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Tue, 28 Mar 2023 14:34:48 +0200 Subject: [PATCH 69/73] Fix add-test for mac --- tests/queries/0_stateless/add-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/add-test b/tests/queries/0_stateless/add-test index e8e68cf174e..dbff8212020 100755 --- a/tests/queries/0_stateless/add-test +++ b/tests/queries/0_stateless/add-test @@ -10,7 +10,7 @@ TESTS_PATH=$(dirname ${BASH_SOURCE[0]}) set -ue # shellcheck disable=SC2010 -LAST_TEST_NO=$(ls -1 ${TESTS_PATH} | grep -P -o '^\d+' | sort -nr | head -1) +LAST_TEST_NO=$(ls -1 ${TESTS_PATH} | grep -E -o '^[0-9]+' | sort -nr | head -1) # remove leading zeros, increment and add padding zeros to 5 digits NEW_TEST_NO=$(printf "%05d\n" $((10#$LAST_TEST_NO + 1))) From c8ec900d905ae662a5ac6c57c8eaafa67e0a0d18 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 28 Mar 2023 15:37:49 +0200 Subject: [PATCH 70/73] Fixed comment for usage on cluster --- docs/en/operations/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 46c3a52b5c7..000c3551fda 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -398,4 +398,4 @@ To disallow concurrent backup/restore, you can use these settings respectively. ``` The default value for both is true, so by default concurrent backup/restores are allowed. -When this setting is used on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file +When these setting are true on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file From afe5982c4ee8cae2531178b30bd3141efade81e7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 28 Mar 2023 15:39:28 +0200 Subject: [PATCH 71/73] Fixed typo - settings --- docs/en/operations/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 000c3551fda..9005432f1bd 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -398,4 +398,4 @@ To disallow concurrent backup/restore, you can use these settings respectively. ``` The default value for both is true, so by default concurrent backup/restores are allowed. -When these setting are true on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file +When these settings are true on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file From a26cfd523c25ae4af7ef43bb552cbcdaa6bef115 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 28 Mar 2023 15:42:21 +0200 Subject: [PATCH 72/73] Updated right bool for disallowing concurrent backups and restores --- docs/en/operations/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 9005432f1bd..a31a52f509e 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -398,4 +398,4 @@ To disallow concurrent backup/restore, you can use these settings respectively. ``` The default value for both is true, so by default concurrent backup/restores are allowed. -When these settings are true on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file +When these settings are false on a cluster, only 1 backup/restore is allowed to run on a cluster at a time. \ No newline at end of file From 0fad5bdc7d6debb2e0da7ca890449022c50d713e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 28 Mar 2023 19:52:51 +0200 Subject: [PATCH 73/73] Add a script to clone or update clickhouse-docs --- docs/.gitignore | 1 + docs/README.md | 2 ++ docs/get-clickhouse-docs.sh | 31 +++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 docs/get-clickhouse-docs.sh diff --git a/docs/.gitignore b/docs/.gitignore index 378eac25d31..509538d9051 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1 +1,2 @@ build +clickhouse-docs diff --git a/docs/README.md b/docs/README.md index 9bfd3d2b897..0cd35a4e3ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,6 +40,8 @@ The documentation contains information about all the aspects of the ClickHouse l At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, and Chinese. We store the reference documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs), and user guides in a separate repo [Clickhouse/clickhouse-docs](https://github.com/ClickHouse/clickhouse-docs). +To get the latter launch the `get-clickhouse-docs.sh` script. + Each language lies in the corresponding folder. Files that are not translated from English are symbolic links to the English ones. diff --git a/docs/get-clickhouse-docs.sh b/docs/get-clickhouse-docs.sh new file mode 100644 index 00000000000..1ba0dae9844 --- /dev/null +++ b/docs/get-clickhouse-docs.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -e +# The script to clone or update the user-guides documentation repo +# https://github.com/ClickHouse/clickhouse-docs + +WORKDIR=$(dirname "$0") +WORKDIR=$(readlink -f "${WORKDIR}") +cd "$WORKDIR" + +if [ -d "clickhouse-docs" ]; then + git -C clickhouse-docs pull +else + if [ -n "$1" ]; then + url_type="$1" + else + read -rp "Enter the URL type (ssh | https): " url_type + fi + case "$url_type" in + ssh) + git_url=git@github.com:ClickHouse/clickhouse-docs.git + ;; + https) + git_url=https://github.com/ClickHouse/clickhouse-docs.git + ;; + *) + echo "Url type must be 'ssh' or 'https'" + exit 1 + ;; + esac + git clone "$git_url" "clickhouse-docs" +fi