mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 16:42:05 +00:00
Merge branch 'master' into Fix--Distinct-combinator-docs
This commit is contained in:
Alsu Giliazova
GitHub
commit
3c05e515c2
@ -16,3 +16,6 @@
|
||||
|
||||
# Applied Black formatter for Python code
|
||||
e6f5a3f98b21ba99cf274a9833797889e020a2b3
|
||||
|
||||
# Enabling clang-tidy readability-else-no-return rule
|
||||
67c1e89d90ef576e62f8b1c68269742a3c6f9b1e
|
||||
|
||||
1
.github/actionlint.yml
vendored
1
.github/actionlint.yml
vendored
@ -4,7 +4,6 @@ self-hosted-runner:
|
||||
- func-tester
|
||||
- func-tester-aarch64
|
||||
- fuzzer-unit-tester
|
||||
- stress-tester
|
||||
- style-checker
|
||||
- style-checker-aarch64
|
||||
- release-maker
|
||||
|
||||
21
.github/workflows/backport_branches.yml
vendored
21
.github/workflows/backport_branches.yml
vendored
@ -229,18 +229,26 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (tsan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
############################# INTEGRATION TESTS #############################################
|
||||
#############################################################################################
|
||||
IntegrationTestsRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
IntegrationTestsAsanOldAnalyzer:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (release)
|
||||
runner_type: stress-tester
|
||||
test_name: Integration tests (asan, old analyzer)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (tsan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FinishCheck:
|
||||
if: ${{ !cancelled() }}
|
||||
@ -250,7 +258,8 @@ jobs:
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- StressTestTsan
|
||||
- IntegrationTestsRelease
|
||||
- IntegrationTestsTsan
|
||||
- IntegrationTestsAsanOldAnalyzer
|
||||
- CompatibilityCheckX86
|
||||
- CompatibilityCheckAarch64
|
||||
runs-on: [self-hosted, style-checker]
|
||||
|
||||
3
.github/workflows/create_release.yml
vendored
3
.github/workflows/create_release.yml
vendored
@ -3,6 +3,9 @@ name: CreateRelease
|
||||
concurrency:
|
||||
group: release
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
||||
4
.github/workflows/docker_test_images.yml
vendored
4
.github/workflows/docker_test_images.yml
vendored
@ -1,4 +1,5 @@
|
||||
name: Build docker images
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
@ -12,6 +13,9 @@ name: Build docker images
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
jobs:
|
||||
DockerBuildAarch64:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
|
||||
18
.github/workflows/release_branches.yml
vendored
18
.github/workflows/release_branches.yml
vendored
@ -374,7 +374,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (asan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
@ -382,7 +382,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (tsan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
@ -390,7 +390,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (msan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestUBsan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
@ -398,7 +398,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (ubsan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
@ -406,7 +406,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (debug)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
############################# INTEGRATION TESTS #############################################
|
||||
@ -417,7 +417,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (asan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsAnalyzerAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
@ -425,7 +425,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (asan, old analyzer)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
@ -433,7 +433,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (tsan)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
@ -441,7 +441,7 @@ jobs:
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (release)
|
||||
runner_type: stress-tester
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FinishCheck:
|
||||
if: ${{ !cancelled() }}
|
||||
|
||||
4
.github/workflows/reusable_build_stage.yml
vendored
4
.github/workflows/reusable_build_stage.yml
vendored
@ -1,7 +1,11 @@
|
||||
### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
|
||||
# and then merge reusable_build_stage.yml and reusable_test_stage.yml
|
||||
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
name: BuildStageWF
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
|
||||
4
.github/workflows/reusable_test_stage.yml
vendored
4
.github/workflows/reusable_test_stage.yml
vendored
@ -1,4 +1,8 @@
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
|
||||
name: StageWF
|
||||
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
|
||||
@ -339,7 +339,6 @@ set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3
|
||||
set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O${DEBUG_O_LEVEL} ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
|
||||
|
||||
if (OS_DARWIN)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main")
|
||||
endif()
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ curl https://clickhouse.com/ | sh
|
||||
|
||||
Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
|
||||
|
||||
* [v24.9 Community Call](https://clickhouse.com/company/events/v24-9-community-release-call) - September 26
|
||||
* [v24.10 Community Call](https://clickhouse.com/company/events/v24-10-community-release-call) - October 31
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
|
||||
@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 24.9 | ✔️ |
|
||||
| 24.8 | ✔️ |
|
||||
| 24.7 | ✔️ |
|
||||
| 24.6 | ✔️ |
|
||||
| 24.6 | ❌ |
|
||||
| 24.5 | ❌ |
|
||||
| 24.4 | ❌ |
|
||||
| 24.3 | ✔️ |
|
||||
|
||||
@ -110,8 +110,7 @@ struct DecomposedFloat
|
||||
{
|
||||
if (!isNegative())
|
||||
return rhs > 0 ? -1 : 1;
|
||||
else
|
||||
return rhs >= 0 ? -1 : 1;
|
||||
return rhs >= 0 ? -1 : 1;
|
||||
}
|
||||
|
||||
/// The case of the most negative integer
|
||||
@ -128,8 +127,7 @@ struct DecomposedFloat
|
||||
|
||||
if (mantissa() == 0)
|
||||
return 0;
|
||||
else
|
||||
return -1;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,9 +167,8 @@ struct DecomposedFloat
|
||||
/// Float has no fractional part means that the numbers are equal.
|
||||
if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0)
|
||||
return 0;
|
||||
else
|
||||
/// Float has fractional part means its abs value is larger.
|
||||
return isNegative() ? -1 : 1;
|
||||
/// Float has fractional part means its abs value is larger.
|
||||
return isNegative() ? -1 : 1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -205,8 +205,7 @@ JSON::ElementType JSON::getType() const
|
||||
Pos after_string = skipString();
|
||||
if (after_string < ptr_end && *after_string == ':')
|
||||
return TYPE_NAME_VALUE_PAIR;
|
||||
else
|
||||
return TYPE_STRING;
|
||||
return TYPE_STRING;
|
||||
}
|
||||
default:
|
||||
throw JSONException(std::string("JSON: unexpected char ") + *ptr_begin + ", expected one of '{[tfn-0123456789\"'");
|
||||
@ -474,8 +473,7 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const
|
||||
|
||||
if (it == end())
|
||||
return nullptr;
|
||||
else
|
||||
return it->data();
|
||||
return it->data();
|
||||
}
|
||||
|
||||
|
||||
@ -487,7 +485,7 @@ bool JSON::hasEscapes() const
|
||||
|
||||
if (*pos == '"')
|
||||
return false;
|
||||
else if (*pos == '\\')
|
||||
if (*pos == '\\')
|
||||
return true;
|
||||
throw JSONException("JSON: unexpected end of data.");
|
||||
}
|
||||
@ -503,7 +501,7 @@ bool JSON::hasSpecialChars() const
|
||||
|
||||
if (*pos == '"')
|
||||
return false;
|
||||
else if (pos < ptr_end)
|
||||
if (pos < ptr_end)
|
||||
return true;
|
||||
throw JSONException("JSON: unexpected end of data.");
|
||||
}
|
||||
@ -682,10 +680,9 @@ double JSON::toDouble() const
|
||||
|
||||
if (type == TYPE_NUMBER)
|
||||
return getDouble();
|
||||
else if (type == TYPE_STRING)
|
||||
if (type == TYPE_STRING)
|
||||
return JSON(ptr_begin + 1, ptr_end, level + 1).getDouble();
|
||||
else
|
||||
throw JSONException("JSON: cannot convert value to double.");
|
||||
throw JSONException("JSON: cannot convert value to double.");
|
||||
}
|
||||
|
||||
Int64 JSON::toInt() const
|
||||
@ -694,10 +691,9 @@ Int64 JSON::toInt() const
|
||||
|
||||
if (type == TYPE_NUMBER)
|
||||
return getInt();
|
||||
else if (type == TYPE_STRING)
|
||||
if (type == TYPE_STRING)
|
||||
return JSON(ptr_begin + 1, ptr_end, level + 1).getInt();
|
||||
else
|
||||
throw JSONException("JSON: cannot convert value to signed integer.");
|
||||
throw JSONException("JSON: cannot convert value to signed integer.");
|
||||
}
|
||||
|
||||
UInt64 JSON::toUInt() const
|
||||
@ -706,10 +702,9 @@ UInt64 JSON::toUInt() const
|
||||
|
||||
if (type == TYPE_NUMBER)
|
||||
return getUInt();
|
||||
else if (type == TYPE_STRING)
|
||||
if (type == TYPE_STRING)
|
||||
return JSON(ptr_begin + 1, ptr_end, level + 1).getUInt();
|
||||
else
|
||||
throw JSONException("JSON: cannot convert value to unsigned integer.");
|
||||
throw JSONException("JSON: cannot convert value to unsigned integer.");
|
||||
}
|
||||
|
||||
std::string JSON::toString() const
|
||||
@ -718,11 +713,9 @@ std::string JSON::toString() const
|
||||
|
||||
if (type == TYPE_STRING)
|
||||
return getString();
|
||||
else
|
||||
{
|
||||
Pos pos = skipElement();
|
||||
return std::string(ptr_begin, pos - ptr_begin);
|
||||
}
|
||||
|
||||
Pos pos = skipElement();
|
||||
return std::string(ptr_begin, pos - ptr_begin);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -203,9 +203,7 @@ T JSON::getWithDefault(const std::string & key, const T & default_) const
|
||||
|
||||
if (key_json.isType<T>())
|
||||
return key_json.get<T>();
|
||||
else
|
||||
return default_;
|
||||
}
|
||||
else
|
||||
return default_;
|
||||
}
|
||||
return default_;
|
||||
}
|
||||
|
||||
@ -151,19 +151,19 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size)
|
||||
return unalignedLoad<uint64_t>(p1) == unalignedLoad<uint64_t>(p2)
|
||||
&& unalignedLoad<uint64_t>(p1 + size - 8) == unalignedLoad<uint64_t>(p2 + size - 8);
|
||||
}
|
||||
else if (size >= 4)
|
||||
if (size >= 4)
|
||||
{
|
||||
/// Chunks of 4..7 bytes.
|
||||
return unalignedLoad<uint32_t>(p1) == unalignedLoad<uint32_t>(p2)
|
||||
&& unalignedLoad<uint32_t>(p1 + size - 4) == unalignedLoad<uint32_t>(p2 + size - 4);
|
||||
}
|
||||
else if (size >= 2)
|
||||
if (size >= 2)
|
||||
{
|
||||
/// Chunks of 2..3 bytes.
|
||||
return unalignedLoad<uint16_t>(p1) == unalignedLoad<uint16_t>(p2)
|
||||
&& unalignedLoad<uint16_t>(p1 + size - 2) == unalignedLoad<uint16_t>(p2 + size - 2);
|
||||
}
|
||||
else if (size >= 1)
|
||||
if (size >= 1)
|
||||
{
|
||||
/// A single byte.
|
||||
return *p1 == *p2;
|
||||
|
||||
@ -53,10 +53,9 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv,
|
||||
key = arg.substr(key_start);
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
key = "";
|
||||
}
|
||||
|
||||
key = "";
|
||||
|
||||
|
||||
if (key_start == std::string::npos)
|
||||
continue;
|
||||
|
||||
@ -330,9 +330,8 @@ inline const char * find_first_symbols_dispatch(const char * begin, const char *
|
||||
#if defined(__SSE4_2__)
|
||||
if (sizeof...(symbols) >= 5)
|
||||
return find_first_symbols_sse42<positive, return_mode, sizeof...(symbols), symbols...>(begin, end);
|
||||
else
|
||||
#endif
|
||||
return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);
|
||||
return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);
|
||||
}
|
||||
|
||||
template <bool positive, ReturnMode return_mode>
|
||||
@ -341,9 +340,8 @@ inline const char * find_first_symbols_dispatch(const std::string_view haystack,
|
||||
#if defined(__SSE4_2__)
|
||||
if (symbols.str.size() >= 5)
|
||||
return find_first_symbols_sse42<positive, return_mode>(haystack.begin(), haystack.end(), symbols);
|
||||
else
|
||||
#endif
|
||||
return find_first_symbols_sse2<positive, return_mode>(haystack.begin(), haystack.end(), symbols.str.data(), symbols.str.size());
|
||||
return find_first_symbols_sse2<positive, return_mode>(haystack.begin(), haystack.end(), symbols.str.data(), symbols.str.size());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,8 +33,7 @@ std::optional<uint64_t> getCgroupsV2MemoryLimit()
|
||||
uint64_t value;
|
||||
if (setting_file >> value)
|
||||
return {value};
|
||||
else
|
||||
return {}; /// e.g. the cgroups default "max"
|
||||
return {}; /// e.g. the cgroups default "max"
|
||||
}
|
||||
current_cgroup = current_cgroup.parent_path();
|
||||
}
|
||||
|
||||
17
ci_v2/docker/style-test/Dockerfile
Normal file
17
ci_v2/docker/style-test/Dockerfile
Normal file
@ -0,0 +1,17 @@
|
||||
# docker build -t clickhouse/style-test .
|
||||
FROM ubuntu:22.04
|
||||
|
||||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
aspell \
|
||||
libxml2-utils \
|
||||
python3-pip \
|
||||
locales \
|
||||
git \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
COPY requirements.txt /
|
||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||
4
ci_v2/docker/style-test/requirements.txt
Normal file
4
ci_v2/docker/style-test/requirements.txt
Normal file
@ -0,0 +1,4 @@
|
||||
requests==2.32.3
|
||||
yamllint==1.26.3
|
||||
codespell==2.2.1
|
||||
https://clickhouse-builds.s3.amazonaws.com/packages/praktika-0.1-py3-none-any.whl
|
||||
410
ci_v2/jobs/check_style.py
Normal file
410
ci_v2/jobs/check_style.py
Normal file
@ -0,0 +1,410 @@
|
||||
import math
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from praktika.result import Result
|
||||
from praktika.utils import Shell, Utils
|
||||
|
||||
NPROC = multiprocessing.cpu_count()
|
||||
|
||||
|
||||
def chunk_list(data, n):
|
||||
"""Split the data list into n nearly equal-sized chunks."""
|
||||
chunk_size = math.ceil(len(data) / n)
|
||||
for i in range(0, len(data), chunk_size):
|
||||
yield data[i : i + chunk_size]
|
||||
|
||||
|
||||
def run_check_concurrent(check_name, check_function, files, nproc=NPROC):
|
||||
stop_watch = Utils.Stopwatch()
|
||||
|
||||
if not files:
|
||||
print(f"File list is empty [{files}]")
|
||||
raise
|
||||
|
||||
file_chunks = list(chunk_list(files, nproc))
|
||||
results = []
|
||||
|
||||
# Run check_function concurrently on each chunk
|
||||
with ProcessPoolExecutor(max_workers=NPROC) as executor:
|
||||
futures = [executor.submit(check_function, chunk) for chunk in file_chunks]
|
||||
# Wait for results and process them (optional)
|
||||
for future in futures:
|
||||
try:
|
||||
res = future.result()
|
||||
if res and res not in results:
|
||||
results.append(res)
|
||||
except Exception as e:
|
||||
results.append(f"Exception in {check_name}: {e}")
|
||||
|
||||
result = Result(
|
||||
name=check_name,
|
||||
status=Result.Status.SUCCESS if not results else Result.Status.FAILED,
|
||||
start_time=stop_watch.start_time,
|
||||
duration=stop_watch.duration,
|
||||
info=f"errors: {results}" if results else "",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_simple_check(check_name, check_function, **kwargs):
|
||||
stop_watch = Utils.Stopwatch()
|
||||
|
||||
error = check_function(**kwargs)
|
||||
|
||||
result = Result(
|
||||
name=check_name,
|
||||
status=Result.Status.SUCCESS if not error else Result.Status.FAILED,
|
||||
start_time=stop_watch.start_time,
|
||||
duration=stop_watch.duration,
|
||||
info=error,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_check(check_name, check_function, files):
|
||||
return run_check_concurrent(check_name, check_function, files, nproc=1)
|
||||
|
||||
|
||||
def check_duplicate_includes(file_path):
|
||||
includes = []
|
||||
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
|
||||
for line in f:
|
||||
if re.match(r"^#include ", line):
|
||||
includes.append(line.strip())
|
||||
|
||||
include_counts = {line: includes.count(line) for line in includes}
|
||||
duplicates = {line: count for line, count in include_counts.items() if count > 1}
|
||||
|
||||
if duplicates:
|
||||
return f"{file_path}: {duplicates}"
|
||||
return ""
|
||||
|
||||
|
||||
def check_whitespaces(file_paths):
|
||||
for file in file_paths:
|
||||
exit_code, out, err = Shell.get_res_stdout_stderr(
|
||||
f'./ci_v2/jobs/scripts/check_style/double_whitespaces.pl "{file}"',
|
||||
verbose=False,
|
||||
)
|
||||
if out or err:
|
||||
return out + " err: " + err
|
||||
return ""
|
||||
|
||||
|
||||
def check_yamllint(file_paths):
|
||||
file_paths = " ".join([f"'{file}'" for file in file_paths])
|
||||
exit_code, out, err = Shell.get_res_stdout_stderr(
|
||||
f"yamllint --config-file=./.yamllint {file_paths}", verbose=False
|
||||
)
|
||||
return out or err
|
||||
|
||||
|
||||
def check_xmllint(file_paths):
|
||||
if not isinstance(file_paths, list):
|
||||
file_paths = [file_paths]
|
||||
file_paths = " ".join([f"'{file}'" for file in file_paths])
|
||||
exit_code, out, err = Shell.get_res_stdout_stderr(
|
||||
f"xmllint --noout --nonet {file_paths}", verbose=False
|
||||
)
|
||||
return out or err
|
||||
|
||||
|
||||
def check_functional_test_cases(files):
|
||||
"""
|
||||
Queries with event_date should have yesterday() not today()
|
||||
NOTE: it is not that accuate, but at least something.
|
||||
"""
|
||||
|
||||
patterns = [
|
||||
re.compile(
|
||||
r"(?i)where.*?\bevent_date\s*(=|>=)\s*today\(\)(?!\s*-\s*1)",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
]
|
||||
|
||||
errors = []
|
||||
for test_case in files:
|
||||
try:
|
||||
with open(test_case, "r", encoding="utf-8", errors="replace") as f:
|
||||
file_content = " ".join(
|
||||
f.read().splitlines()
|
||||
) # Combine lines into a single string
|
||||
|
||||
# Check if any pattern matches in the concatenated string
|
||||
if any(pattern.search(file_content) for pattern in patterns):
|
||||
errors.append(
|
||||
f"event_date should be filtered using >=yesterday() in {test_case} (to avoid flakiness)"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f"Error checking {test_case}: {e}")
|
||||
|
||||
for test_case in files:
|
||||
if "fail" in test_case:
|
||||
errors.append(f"test case {test_case} includes 'fail' in its name")
|
||||
|
||||
return " ".join(errors)
|
||||
|
||||
|
||||
def check_gaps_in_tests_numbers(file_paths, gap_threshold=100):
|
||||
test_numbers = set()
|
||||
|
||||
pattern = re.compile(r"(\d+)")
|
||||
|
||||
for file in file_paths:
|
||||
file_name = os.path.basename(file)
|
||||
match = pattern.search(file_name)
|
||||
if match:
|
||||
test_numbers.add(int(match.group(1)))
|
||||
|
||||
sorted_numbers = sorted(test_numbers)
|
||||
large_gaps = []
|
||||
for i in range(1, len(sorted_numbers)):
|
||||
prev_num = sorted_numbers[i - 1]
|
||||
next_num = sorted_numbers[i]
|
||||
diff = next_num - prev_num
|
||||
if diff >= gap_threshold:
|
||||
large_gaps.append(f"Gap ({prev_num}, {next_num}) > {gap_threshold}")
|
||||
|
||||
return large_gaps
|
||||
|
||||
|
||||
def check_broken_links(path, exclude_paths):
|
||||
broken_symlinks = []
|
||||
|
||||
for path in Path(path).rglob("*"):
|
||||
if any(exclude_path in str(path) for exclude_path in exclude_paths):
|
||||
continue
|
||||
if path.is_symlink():
|
||||
if not path.exists():
|
||||
broken_symlinks.append(str(path))
|
||||
|
||||
if broken_symlinks:
|
||||
for symlink in broken_symlinks:
|
||||
print(symlink)
|
||||
return f"Broken symlinks found: {broken_symlinks}"
|
||||
else:
|
||||
return ""
|
||||
|
||||
|
||||
def check_cpp_code():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check_cpp.sh"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_repo_submodules():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check_submodules.sh"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_other():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/checks_to_refactor.sh"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_codespell():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check_typos.sh"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_aspell():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check_aspell.sh"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_mypy():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check-mypy"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_pylint():
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
"./ci_v2/jobs/scripts/check_style/check-pylint"
|
||||
)
|
||||
if err:
|
||||
out += err
|
||||
return out
|
||||
|
||||
|
||||
def check_file_names(files):
|
||||
files_set = set()
|
||||
for file in files:
|
||||
file_ = file.lower()
|
||||
if file_ in files_set:
|
||||
return f"Non-uniq file name in lower case: {file}"
|
||||
files_set.add(file_)
|
||||
return ""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
results = []
|
||||
stop_watch = Utils.Stopwatch()
|
||||
|
||||
all_files = Utils.traverse_paths(
|
||||
include_paths=["."],
|
||||
exclude_paths=[
|
||||
"./.git",
|
||||
"./contrib",
|
||||
"./build",
|
||||
],
|
||||
not_exists_ok=True, # ./build may exist if runs locally
|
||||
)
|
||||
|
||||
cpp_files = Utils.traverse_paths(
|
||||
include_paths=["./src", "./base", "./programs", "./utils"],
|
||||
exclude_paths=[
|
||||
"./base/glibc-compatibility",
|
||||
"./contrib/consistent-hashing",
|
||||
"./base/widechar_width",
|
||||
],
|
||||
file_suffixes=[".h", ".cpp"],
|
||||
)
|
||||
|
||||
yaml_workflow_files = Utils.traverse_paths(
|
||||
include_paths=["./.github"],
|
||||
exclude_paths=[],
|
||||
file_suffixes=[".yaml", ".yml"],
|
||||
)
|
||||
|
||||
xml_files = Utils.traverse_paths(
|
||||
include_paths=["."],
|
||||
exclude_paths=["./.git", "./contrib/"],
|
||||
file_suffixes=[".xml"],
|
||||
)
|
||||
|
||||
functional_test_files = Utils.traverse_paths(
|
||||
include_paths=["./tests/queries"],
|
||||
exclude_paths=[],
|
||||
file_suffixes=[".sql", ".sh", ".py", ".j2"],
|
||||
)
|
||||
|
||||
results.append(
|
||||
Result(
|
||||
name="Read Files",
|
||||
status=Result.Status.SUCCESS,
|
||||
start_time=stop_watch.start_time,
|
||||
duration=stop_watch.duration,
|
||||
)
|
||||
)
|
||||
|
||||
results.append(
|
||||
run_check_concurrent(
|
||||
check_name="Whitespace Check",
|
||||
check_function=check_whitespaces,
|
||||
files=cpp_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_check_concurrent(
|
||||
check_name="YamlLint Check",
|
||||
check_function=check_yamllint,
|
||||
files=yaml_workflow_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_check_concurrent(
|
||||
check_name="XmlLint Check",
|
||||
check_function=check_xmllint,
|
||||
files=xml_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_check_concurrent(
|
||||
check_name="Functional Tests scripts smoke check",
|
||||
check_function=check_functional_test_cases,
|
||||
files=functional_test_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_check(
|
||||
check_name="Check Tests Numbers",
|
||||
check_function=check_gaps_in_tests_numbers,
|
||||
files=functional_test_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check Broken Symlinks",
|
||||
check_function=check_broken_links,
|
||||
path="./",
|
||||
exclude_paths=["contrib/", "metadata/", "programs/server/data"],
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check CPP code",
|
||||
check_function=check_cpp_code,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check Submodules",
|
||||
check_function=check_repo_submodules,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_check(
|
||||
check_name="Check File Names",
|
||||
check_function=check_file_names,
|
||||
files=all_files,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check Many Different Things",
|
||||
check_function=check_other,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check Codespell",
|
||||
check_function=check_codespell,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
run_simple_check(
|
||||
check_name="Check Aspell",
|
||||
check_function=check_aspell,
|
||||
)
|
||||
)
|
||||
|
||||
res = Result.create_from(results=results, stopwatch=stop_watch).dump()
|
||||
|
||||
if not res.is_ok():
|
||||
print("Style check: failed")
|
||||
for result in results:
|
||||
if not result.is_ok():
|
||||
print("Failed check:")
|
||||
print(" | ", result)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Style check: ok")
|
||||
3133
ci_v2/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt
Normal file
3133
ci_v2/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt
Normal file
File diff suppressed because it is too large
Load Diff
59
ci_v2/jobs/scripts/check_style/check_aspell.sh
Executable file
59
ci_v2/jobs/scripts/check_style/check_aspell.sh
Executable file
@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# force-enable double star globbing
|
||||
shopt -s globstar
|
||||
|
||||
# Perform spell checking on the docs
|
||||
|
||||
if [[ ${1:-} == "--help" ]] || [[ ${1:-} == "-h" ]]; then
|
||||
echo "Usage $0 [--help|-h] [-i [filename]]"
|
||||
echo " --help|-h: print this help"
|
||||
echo " -i: interactive mode. If filename is specified, check only this file, otherwise check all files"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ROOT_PATH="."
|
||||
|
||||
CHECK_LANG=en
|
||||
|
||||
ASPELL_IGNORE_PATH="${ROOT_PATH}/utils/check-style/aspell-ignore/${CHECK_LANG}"
|
||||
|
||||
if [[ ${1:-} == "-i" ]]; then
|
||||
if [[ ! -z ${2:-} ]]; then
|
||||
FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/${2}
|
||||
else
|
||||
FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md
|
||||
fi
|
||||
for fname in ${FILES_TO_CHECK}; do
|
||||
echo "Checking $fname"
|
||||
aspell --personal=aspell-dict.txt --add-sgml-skip=code --encoding=utf-8 --mode=markdown -W 3 --lang=${CHECK_LANG} --home-dir=${ASPELL_IGNORE_PATH} -c "$fname"
|
||||
done
|
||||
exit
|
||||
fi
|
||||
|
||||
STATUS=0
|
||||
for fname in ${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md; do
|
||||
errors=$(cat "$fname" \
|
||||
| aspell list \
|
||||
-W 3 \
|
||||
--personal=aspell-dict.txt \
|
||||
--add-sgml-skip=code \
|
||||
--encoding=utf-8 \
|
||||
--mode=markdown \
|
||||
--lang=${CHECK_LANG} \
|
||||
--home-dir=${ASPELL_IGNORE_PATH} \
|
||||
| sort | uniq)
|
||||
if [ ! -z "$errors" ]; then
|
||||
STATUS=1
|
||||
echo "====== $fname ======"
|
||||
echo "$errors"
|
||||
fi
|
||||
done
|
||||
|
||||
if (( STATUS != 0 )); then
|
||||
echo "====== Errors found ======"
|
||||
echo "To exclude some words add them to the dictionary file \"${ASPELL_IGNORE_PATH}/aspell-dict.txt\""
|
||||
echo "You can also run ${0} -i to see the errors interactively and fix them or add to the dictionary file"
|
||||
fi
|
||||
|
||||
exit ${STATUS}
|
||||
339
ci_v2/jobs/scripts/check_style/check_cpp.sh
Executable file
339
ci_v2/jobs/scripts/check_style/check_cpp.sh
Executable file
@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# For code formatting we have clang-format.
|
||||
#
|
||||
# But it's not sane to apply clang-format for whole code base,
|
||||
# because it sometimes makes worse for properly formatted files.
|
||||
#
|
||||
# It's only reasonable to blindly apply clang-format only in cases
|
||||
# when the code is likely to be out of style.
|
||||
#
|
||||
# For this purpose we have a script that will use very primitive heuristics
|
||||
# (simple regexps) to check if the code is likely to have basic style violations.
|
||||
# and then to run formatter only for the specified files.
|
||||
|
||||
LC_ALL="en_US.UTF-8"
|
||||
ROOT_PATH="."
|
||||
EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|utils/keeper-bench/example.yaml'
|
||||
|
||||
# From [1]:
|
||||
# But since array_to_string_internal() in array.c still loops over array
|
||||
# elements and concatenates them into a string, it's probably not more
|
||||
# efficient than the looping solutions proposed, but it's more readable.
|
||||
#
|
||||
# [1]: https://stackoverflow.com/a/15394738/328260
|
||||
function in_array()
|
||||
{
|
||||
local IFS="|"
|
||||
local value=$1 && shift
|
||||
|
||||
[[ "${IFS}${*}${IFS}" =~ "${IFS}${value}${IFS}" ]]
|
||||
}
|
||||
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep $@ -P '((class|struct|namespace|enum|if|for|while|else|throw|switch).*|\)(\s*const)?(\s*override)?\s*)\{$|\s$|^ {1,3}[^\* ]\S|\t|^\s*(if|else if|if constexpr|else if constexpr|for|while|catch|switch)\(|\( [^\s\\]|\S \)' |
|
||||
# a curly brace not in a new line, but not for the case of C++11 init or agg. initialization | trailing whitespace | number of ws not a multiple of 4, but not in the case of comment continuation | missing whitespace after for/if/while... before opening brace | whitespaces inside braces
|
||||
grep -v -P '(//|:\s+\*|\$\(\()| \)"'
|
||||
# single-line comment | continuation of a multiline comment | a typical piece of embedded shell code | something like ending of raw string literal
|
||||
|
||||
# Tabs
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep $@ -F $'\t'
|
||||
|
||||
# // namespace comments are unneeded
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep $@ -P '}\s*//+\s*namespace\s*'
|
||||
|
||||
# Broken symlinks
|
||||
find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found"
|
||||
|
||||
# Duplicated or incorrect setting declarations
|
||||
SETTINGS_FILE=$(mktemp)
|
||||
cat $ROOT_PATH/src/Core/Settings.cpp $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " " substr($1, 3, length($1) - 3) " SettingsDeclaration" }' > ${SETTINGS_FILE}
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep "extern const Settings" -T | awk '{print substr($5, 0, length($5) -1) " " substr($4, 9) " " substr($1, 0, length($1) - 1)}' >> ${SETTINGS_FILE}
|
||||
|
||||
# Duplicate extern declarations for settings
|
||||
awk '{if (seen[$0]++) print $3 " -> " $1 ;}' ${SETTINGS_FILE} | while read line;
|
||||
do
|
||||
echo "Found duplicated setting declaration in: $line"
|
||||
done
|
||||
|
||||
# Incorrect declarations for settings
|
||||
for setting in $(awk '{print $1 " " $2}' ${SETTINGS_FILE} | sort | uniq | awk '{ print $1 }' | sort | uniq -d);
|
||||
do
|
||||
expected=$(grep "^$setting " ${SETTINGS_FILE} | grep SettingsDeclaration | awk '{ print $2 }')
|
||||
grep "^$setting " ${SETTINGS_FILE} | grep -v " $expected" | awk '{ print $3 " found setting " $1 " with type " $2 }' | while read line;
|
||||
do
|
||||
echo "In $line but it should be $expected"
|
||||
done
|
||||
done
|
||||
|
||||
rm ${SETTINGS_FILE}
|
||||
|
||||
# Unused/Undefined/Duplicates ErrorCodes/ProfileEvents/CurrentMetrics
|
||||
declare -A EXTERN_TYPES
|
||||
EXTERN_TYPES[ErrorCodes]=int
|
||||
EXTERN_TYPES[ProfileEvents]=Event
|
||||
EXTERN_TYPES[CurrentMetrics]=Metric
|
||||
|
||||
EXTERN_TYPES_EXCLUDES=(
|
||||
ProfileEvents::global_counters
|
||||
ProfileEvents::Event
|
||||
ProfileEvents::Count
|
||||
ProfileEvents::Counters
|
||||
ProfileEvents::end
|
||||
ProfileEvents::increment
|
||||
ProfileEvents::incrementForLogMessage
|
||||
ProfileEvents::getName
|
||||
ProfileEvents::Timer
|
||||
ProfileEvents::Type
|
||||
ProfileEvents::TypeEnum
|
||||
ProfileEvents::dumpToMapColumn
|
||||
ProfileEvents::getProfileEvents
|
||||
ProfileEvents::ThreadIdToCountersSnapshot
|
||||
ProfileEvents::LOCAL_NAME
|
||||
ProfileEvents::keeper_profile_events
|
||||
ProfileEvents::CountersIncrement
|
||||
|
||||
CurrentMetrics::add
|
||||
CurrentMetrics::sub
|
||||
CurrentMetrics::get
|
||||
CurrentMetrics::set
|
||||
CurrentMetrics::end
|
||||
CurrentMetrics::Increment
|
||||
CurrentMetrics::Metric
|
||||
CurrentMetrics::values
|
||||
CurrentMetrics::Value
|
||||
CurrentMetrics::keeper_metrics
|
||||
|
||||
ErrorCodes::ErrorCode
|
||||
ErrorCodes::getName
|
||||
ErrorCodes::increment
|
||||
ErrorCodes::end
|
||||
ErrorCodes::values
|
||||
ErrorCodes::values[i]
|
||||
ErrorCodes::getErrorCodeByName
|
||||
ErrorCodes::Value
|
||||
)
|
||||
for extern_type in ${!EXTERN_TYPES[@]}; do
|
||||
type_of_extern=${EXTERN_TYPES[$extern_type]}
|
||||
allowed_chars='[_A-Za-z]+'
|
||||
|
||||
# Unused
|
||||
# NOTE: to fix automatically, replace echo with:
|
||||
# sed -i "/extern const $type_of_extern $val/d" $file
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
|
||||
# NOTE: the check is pretty dumb and distinguish only by the type_of_extern,
|
||||
# and this matches with zkutil::CreateMode
|
||||
grep -v -e 'src/Common/ZooKeeper/Types.h' -e 'src/Coordination/KeeperConstants.cpp'
|
||||
} | {
|
||||
grep -vP $EXCLUDE_DIRS | xargs grep -l -P "extern const $type_of_extern $allowed_chars"
|
||||
} | while read file; do
|
||||
grep -P "extern const $type_of_extern $allowed_chars;" $file | sed -r -e "s/^.*?extern const $type_of_extern ($allowed_chars);.*?$/\1/" | while read val; do
|
||||
if ! grep -q "$extern_type::$val" $file; then
|
||||
# Excludes for SOFTWARE_EVENT/HARDWARE_EVENT/CACHE_EVENT in ThreadProfileEvents.cpp
|
||||
if [[ ! $extern_type::$val =~ ProfileEvents::Perf.* ]]; then
|
||||
echo "$extern_type::$val is defined but not used in file $file"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Undefined
|
||||
# NOTE: to fix automatically, replace echo with:
|
||||
# ( grep -q -F 'namespace $extern_type' $file && \
|
||||
# sed -i -r "0,/(\s*)extern const $type_of_extern [$allowed_chars]+/s//\1extern const $type_of_extern $val;\n&/" $file || \
|
||||
# awk '{ print; if (ns == 1) { ns = 2 }; if (ns == 2) { ns = 0; print "namespace $extern_type\n{\n extern const $type_of_extern '$val';\n}" } }; /namespace DB/ { ns = 1; };' < $file > ${file}.tmp && mv ${file}.tmp $file )
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
|
||||
grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars"
|
||||
} | while read file; do
|
||||
grep -P "$extern_type::$allowed_chars" $file | grep -P -v '^\s*//' | sed -r -e "s/^.*?$extern_type::($allowed_chars).*?$/\1/" | while read val; do
|
||||
if ! grep -q "extern const $type_of_extern $val" $file; then
|
||||
if ! in_array "$extern_type::$val" "${EXTERN_TYPES_EXCLUDES[@]}"; then
|
||||
echo "$extern_type::$val is used in file $file but not defined"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Duplicates
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | {
|
||||
grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars"
|
||||
} | while read file; do
|
||||
grep -P "extern const $type_of_extern $allowed_chars;" $file | sort | uniq -c | grep -v -P ' +1 ' && echo "Duplicate $extern_type in file $file"
|
||||
done
|
||||
done
|
||||
|
||||
# Three or more consecutive empty lines
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
while read file; do awk '/^$/ { ++i; if (i > 2) { print "More than two consecutive empty lines in file '$file'" } } /./ { i = 0 }' $file; done
|
||||
|
||||
# Check that every header file has #pragma once in first line
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
while read file; do [[ $(head -n1 $file) != '#pragma once' ]] && echo "File $file must have '#pragma once' in first line"; done
|
||||
|
||||
# Too many exclamation marks
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -F '!!!' | grep -P '.' && echo "Too many exclamation marks (looks dirty, unconfident)."
|
||||
|
||||
# Exclamation mark in a message
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -F '!",' | grep -P '.' && echo "No need for an exclamation mark (looks dirty, unconfident)."
|
||||
|
||||
# Trailing whitespaces
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -n -P ' $' | grep -n -P '.' && echo "^ Trailing whitespaces."
|
||||
|
||||
# Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -P 'std::[io]?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream"
|
||||
|
||||
# Forbid std::cerr/std::cout in src (fine in programs/utils)
|
||||
std_cerr_cout_excludes=(
|
||||
/examples/
|
||||
/tests/
|
||||
_fuzzer
|
||||
# OK
|
||||
src/Common/ProgressIndication.cpp
|
||||
# only under #ifdef DBMS_HASH_MAP_DEBUG_RESIZES, that is used only in tests
|
||||
src/Common/HashTable/HashTable.h
|
||||
# SensitiveDataMasker::printStats()
|
||||
src/Common/SensitiveDataMasker.cpp
|
||||
# StreamStatistics::print()
|
||||
src/Compression/LZ4_decompress_faster.cpp
|
||||
# ContextSharedPart with subsequent std::terminate()
|
||||
src/Interpreters/Context.cpp
|
||||
# IProcessor::dump()
|
||||
src/Processors/IProcessor.cpp
|
||||
src/Client/ClientApplicationBase.cpp
|
||||
src/Client/ClientBase.cpp
|
||||
src/Client/LineReader.cpp
|
||||
src/Client/QueryFuzzer.cpp
|
||||
src/Client/Suggest.cpp
|
||||
src/Client/ClientBase.h
|
||||
src/Client/LineReader.h
|
||||
src/Client/ReplxxLineReader.h
|
||||
src/Bridge/IBridge.cpp
|
||||
src/Daemon/BaseDaemon.cpp
|
||||
src/Loggers/Loggers.cpp
|
||||
src/Common/GWPAsan.cpp
|
||||
src/Common/ProgressIndication.h
|
||||
)
|
||||
sources_with_std_cerr_cout=( $(
|
||||
find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \
|
||||
grep -vP $EXCLUDE_DIRS | \
|
||||
grep -F -v $(printf -- "-e %s " "${std_cerr_cout_excludes[@]}") | \
|
||||
xargs grep -F --with-filename -e std::cerr -e std::cout | cut -d: -f1 | sort -u
|
||||
) )
|
||||
|
||||
# Exclude comments
|
||||
for src in "${sources_with_std_cerr_cout[@]}"; do
|
||||
# suppress stderr, since it may contain warning for #pargma once in headers
|
||||
if gcc -fpreprocessed -dD -E "$src" 2>/dev/null | grep -F -q -e std::cerr -e std::cout; then
|
||||
echo "$src: uses std::cerr/std::cout"
|
||||
fi
|
||||
done
|
||||
|
||||
expect_tests=( $(find $ROOT_PATH/tests/queries -name '*.expect') )
|
||||
for test_case in "${expect_tests[@]}"; do
|
||||
pattern="^exp_internal -f \$CLICKHOUSE_TMP/\$basename.debuglog 0$"
|
||||
grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
|
||||
|
||||
if grep -q "^spawn.*CLICKHOUSE_CLIENT_BINARY$" "$test_case"; then
|
||||
pattern="^spawn.*CLICKHOUSE_CLIENT_BINARY.*--history_file$"
|
||||
grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
|
||||
fi
|
||||
|
||||
# Otherwise expect_after/expect_before will not bail without stdin attached
|
||||
# (and actually this is a hack anyway, correct way is to use $any_spawn_id)
|
||||
pattern="-i \$any_spawn_id timeout"
|
||||
grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
|
||||
pattern="-i \$any_spawn_id eof"
|
||||
grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'"
|
||||
done
|
||||
|
||||
# Forbid non-unique error codes
|
||||
if [[ "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | wc -l)" != "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | sort | uniq | wc -l)" ]]
|
||||
then
|
||||
echo "ErrorCodes.cpp contains non-unique error codes"
|
||||
fi
|
||||
|
||||
# Check that there is no system-wide libraries/headers in use.
|
||||
#
|
||||
# NOTE: it is better to override find_path/find_library in cmake, but right now
|
||||
# it is not possible, see [1] for the reference.
|
||||
#
|
||||
# [1]: git grep --recurse-submodules -e find_library -e find_path contrib
|
||||
if git grep -e find_path -e find_library -- :**CMakeLists.txt; then
|
||||
echo "There is find_path/find_library usage. ClickHouse should use everything bundled. Consider adding one more contrib module."
|
||||
fi
|
||||
|
||||
# Forbid std::filesystem::is_symlink and std::filesystem::read_symlink, because it's easy to use them incorrectly
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -P '::(is|read)_symlink' | grep -v "STYLE_CHECK_ALLOW_STD_FS_SYMLINK" && echo "Use DB::FS::isSymlink and DB::FS::readSymlink instead"
|
||||
|
||||
# Forbid __builtin_unreachable(), because it's hard to debug when it becomes reachable
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -P '__builtin_unreachable' && echo "Use UNREACHABLE() from defines.h instead"
|
||||
|
||||
# Forbid mt19937() and random_device() which are outdated and slow
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -P '(std::mt19937|std::mersenne_twister_engine|std::random_device)' && echo "Use pcg64_fast (from pcg_random.h) and randomSeed (from Common/randomSeed.h) instead"
|
||||
|
||||
# Require checking return value of close(),
|
||||
# since it can hide fd misuse and break other places.
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -e ' close(.*fd' -e ' ::close(' | grep -v = && echo "Return value of close() should be checked"
|
||||
|
||||
# A small typo can lead to debug code in release builds, see https://github.com/ClickHouse/ClickHouse/pull/47647
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -l -F '#ifdef NDEBUG' | xargs -I@FILE awk '/#ifdef NDEBUG/ { inside = 1; dirty = 1 } /#endif/ { if (inside && dirty) { print "File @FILE has suspicious #ifdef NDEBUG, possibly confused with #ifndef NDEBUG" }; inside = 0 } /#else/ { dirty = 0 }' @FILE
|
||||
|
||||
# If a user is doing dynamic or typeid cast with a pointer, and immediately dereferencing it, it is unsafe.
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep --line-number -P '(dynamic|typeid)_cast<[^>]+\*>\([^\(\)]+\)->' | grep -P '.' && echo "It's suspicious when you are doing a dynamic_cast or typeid_cast with a pointer and immediately dereferencing it. Use references instead of pointers or check a pointer to nullptr."
|
||||
|
||||
# Check for bad punctuation: whitespace before comma.
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number '\w ,' | grep -v 'bad punctuation is ok here' && echo "^ There is bad punctuation: whitespace before comma. You should write it like this: 'Hello, world!'"
|
||||
|
||||
# Check usage of std::regex which is too bloated and slow.
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number 'std::regex' | grep -P '.' && echo "^ Please use re2 instead of std::regex"
|
||||
|
||||
# Cyrillic characters hiding inside Latin.
|
||||
find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | grep -v StorageSystemContributors.generated.cpp | xargs grep -P --line-number '[a-zA-Z][а-яА-ЯёЁ]|[а-яА-ЯёЁ][a-zA-Z]' && echo "^ Cyrillic characters found in unexpected place."
|
||||
|
||||
# Orphaned header files.
|
||||
join -v1 <(find $ROOT_PATH/{src,programs,utils} -name '*.h' -printf '%f\n' | sort | uniq) <(find $ROOT_PATH/{src,programs,utils} -name '*.cpp' -or -name '*.c' -or -name '*.h' -or -name '*.S' | xargs grep --no-filename -o -P '[\w-]+\.h' | sort | uniq) |
|
||||
grep . && echo '^ Found orphan header files.'
|
||||
|
||||
# Don't allow dynamic compiler check with CMake, because we are using hermetic, reproducible, cross-compiled, static (TLDR, good) builds.
|
||||
ls -1d $ROOT_PATH/contrib/*-cmake | xargs -I@ find @ -name 'CMakeLists.txt' -or -name '*.cmake' | xargs grep --with-filename -i -P 'check_c_compiler_flag|check_cxx_compiler_flag|check_c_source_compiles|check_cxx_source_compiles|check_include_file|check_symbol_exists|cmake_push_check_state|cmake_pop_check_state|find_package|CMAKE_REQUIRED_FLAGS|CheckIncludeFile|CheckCCompilerFlag|CheckCXXCompilerFlag|CheckCSourceCompiles|CheckCXXSourceCompiles|CheckCSymbolExists|CheckCXXSymbolExists' | grep -v Rust && echo "^ It's not allowed to have dynamic compiler checks with CMake."
|
||||
|
||||
# Wrong spelling of abbreviations, e.g. SQL is right, Sql is wrong. XMLHttpRequest is very wrong.
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -P 'Sql|Html|Xml|Cpu|Tcp|Udp|Http|Db|Json|Yaml' | grep -v -P 'RabbitMQ|Azure|Aws|aws|Avro|IO/S3' &&
|
||||
echo "Abbreviations such as SQL, XML, HTTP, should be in all caps. For example, SQL is right, Sql is wrong. XMLHttpRequest is very wrong."
|
||||
|
||||
find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep -F -i 'ErrorCodes::LOGICAL_ERROR, "Logical error:' &&
|
||||
echo "If an exception has LOGICAL_ERROR code, there is no need to include the text 'Logical error' in the exception message, because then the phrase 'Logical error' will be printed twice."
|
||||
|
||||
# There shouldn't be any code snippets under GPL or LGPL
|
||||
find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL"
|
||||
|
||||
PATTERN="allow_";
|
||||
DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.cpp | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u));
|
||||
[ -n "$DIFF" ] && echo "$DIFF" && echo "^^ Detected 'allow_*' settings that might need to be included in src/Databases/enableAllExperimentalSettings.cpp" && echo "Alternatively, consider adding an exception to utils/check-style/experimental_settings_ignore.txt"
|
||||
37
ci_v2/jobs/scripts/check_style/check_submodules.sh
Executable file
37
ci_v2/jobs/scripts/check_style/check_submodules.sh
Executable file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# The script checks if all submodules defined in $GIT_ROOT/.gitmodules exist in $GIT_ROOT/contrib
|
||||
|
||||
set -e
|
||||
|
||||
GIT_ROOT="."
|
||||
|
||||
cd "$GIT_ROOT"
|
||||
|
||||
# Remove keys for submodule.*.path parameters, the values are separated by \0
|
||||
# and check if the directory exists
|
||||
git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \
|
||||
xargs -P100 -0 --no-run-if-empty -I{} bash -c 'if ! test -d '"'{}'"'; then echo Directory for submodule {} is not found; exit 1; fi' 2>&1
|
||||
|
||||
|
||||
# And check that the submodule is fine
|
||||
git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \
|
||||
xargs -P100 -0 --no-run-if-empty -I{} git submodule status -q '{}' 2>&1
|
||||
|
||||
|
||||
# All submodules should be from https://github.com/
|
||||
git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.url' | \
|
||||
while read -r line; do
|
||||
name=${line#submodule.}; name=${name%.url*}
|
||||
url=${line#* }
|
||||
[[ "$url" != 'https://github.com/'* ]] && echo "All submodules should be from https://github.com/, submodule '$name' has '$url'"
|
||||
done
|
||||
|
||||
# All submodules should be of this form: [submodule "contrib/libxyz"] (for consistency, the submodule name does matter too much)
|
||||
# - restrict the check to top-level .gitmodules file
|
||||
git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.path' | \
|
||||
while read -r line; do
|
||||
name=${line#submodule.}; name=${name%.path*}
|
||||
path=${line#* }
|
||||
[ "$name" != "$path" ] && echo "Submodule name '$name' is not equal to it's path '$path'"
|
||||
done
|
||||
15
ci_v2/jobs/scripts/check_style/check_typos.sh
Executable file
15
ci_v2/jobs/scripts/check_style/check_typos.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Check for typos in code.
|
||||
|
||||
ROOT_PATH="."
|
||||
|
||||
#FIXME: check all (or almost all) repo
|
||||
codespell \
|
||||
--skip "*generated*,*gperf*,*.bin,*.mrk*,*.idx,checksums.txt,*.dat,*.pyc,*.kate-swp,*obfuscateQueries.cpp,d3-*.js,*.min.js,*.sum,${ROOT_PATH}/utils/check-style/aspell-ignore" \
|
||||
--ignore-words "${ROOT_PATH}/utils/check-style/codespell-ignore-words.list" \
|
||||
--exclude-file "${ROOT_PATH}/utils/check-style/codespell-ignore-lines.list" \
|
||||
--quiet-level 2 \
|
||||
"$ROOT_PATH"/{src,base,programs,utils} \
|
||||
$@ | grep -P '.' \
|
||||
&& echo -e "\nFound some typos in code.\nSee the files utils/check-style/codespell* if you want to add an exception."
|
||||
98
ci_v2/jobs/scripts/check_style/checks_to_refactor.sh
Executable file
98
ci_v2/jobs/scripts/check_style/checks_to_refactor.sh
Executable file
@ -0,0 +1,98 @@
|
||||
#!/bin/bash
|
||||
|
||||
ROOT_PATH="."
|
||||
|
||||
# Queries to system.query_log/system.query_thread_log should have current_database = currentDatabase() condition
|
||||
# NOTE: it is not that accurate, but at least something.
|
||||
tests_with_query_log=( $(
|
||||
find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
|
||||
xargs grep --with-filename -e system.query_log -e system.query_thread_log | cut -d: -f1 | sort -u
|
||||
) )
|
||||
for test_case in "${tests_with_query_log[@]}"; do
|
||||
grep -qE current_database.*currentDatabase "$test_case" || {
|
||||
grep -qE 'current_database.*\$CLICKHOUSE_DATABASE' "$test_case"
|
||||
} || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case"
|
||||
done
|
||||
|
||||
grep -iE 'SYSTEM STOP MERGES;?$' -R $ROOT_PATH/tests/queries && echo "Merges cannot be disabled globally in fast/stateful/stateless tests, because it will break concurrently running queries"
|
||||
|
||||
|
||||
# Queries to:
|
||||
tables_with_database_column=(
|
||||
system.tables
|
||||
system.parts
|
||||
system.detached_parts
|
||||
system.parts_columns
|
||||
system.columns
|
||||
system.projection_parts
|
||||
system.mutations
|
||||
)
|
||||
# should have database = currentDatabase() condition
|
||||
#
|
||||
# NOTE: it is not that accuate, but at least something.
|
||||
tests_with_database_column=( $(
|
||||
find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
|
||||
xargs grep --with-filename $(printf -- "-e %s " "${tables_with_database_column[@]}") |
|
||||
grep -v -e ':--' -e ':#' |
|
||||
cut -d: -f1 | sort -u
|
||||
) )
|
||||
for test_case in "${tests_with_database_column[@]}"; do
|
||||
grep -qE database.*currentDatabase "$test_case" || {
|
||||
grep -qE 'database.*\$CLICKHOUSE_DATABASE' "$test_case"
|
||||
} || {
|
||||
# explicit database
|
||||
grep -qE "database[ ]*=[ ]*'" "$test_case"
|
||||
} || {
|
||||
echo "Queries to ${tables_with_database_column[*]} does not have database = currentDatabase()/\$CLICKHOUSE_DATABASE condition in $test_case"
|
||||
}
|
||||
done
|
||||
|
||||
# Queries with ReplicatedMergeTree
|
||||
# NOTE: it is not that accuate, but at least something.
|
||||
tests_with_replicated_merge_tree=( $(
|
||||
find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
|
||||
xargs grep --with-filename -e "Replicated.*MergeTree[ ]*(.*" | cut -d: -f1 | sort -u
|
||||
) )
|
||||
for test_case in "${tests_with_replicated_merge_tree[@]}"; do
|
||||
case "$test_case" in
|
||||
*.gen.*)
|
||||
;;
|
||||
*.sh)
|
||||
test_case_zk_prefix="\(\$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX\|{database}\)"
|
||||
grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)"
|
||||
;;
|
||||
*.sql|*.sql.j2)
|
||||
test_case_zk_prefix="\({database}\|currentDatabase()\|{uuid}\|{default_path_test}\)"
|
||||
grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)"
|
||||
;;
|
||||
*.py)
|
||||
# Right now there is not such tests anyway
|
||||
echo "No ReplicatedMergeTree style check for *.py ($test_case)"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# The stateful directory should only contain the tests that depend on the test dataset (hits or visits).
|
||||
find $ROOT_PATH/tests/queries/1_stateful -name '*.sql' -or -name '*.sh' | grep -v '00076_system_columns_bytes' | xargs -I{} bash -c 'grep -q -P "hits|visits" "{}" || echo "The test {} does not depend on the test dataset (hits or visits table) and should be located in the 0_stateless directory. You can also add an exception to the check-style script."'
|
||||
|
||||
# Check for existence of __init__.py files
|
||||
for i in "${ROOT_PATH}"/tests/integration/test_*; do FILE="${i}/__init__.py"; [ ! -f "${FILE}" ] && echo "${FILE} should exist for every integration test"; done
|
||||
|
||||
# Check for executable bit on non-executable files
|
||||
find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.j2' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -P '.' && echo "These files should not be executable."
|
||||
|
||||
# Check for BOM
|
||||
find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' | grep -P '.' && echo "Files should not have UTF-8 BOM"
|
||||
find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' | grep -P '.' && echo "Files should not have UTF-16LE BOM"
|
||||
find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' | grep -P '.' && echo "Files should not have UTF-16BE BOM"
|
||||
|
||||
# Conflict markers
|
||||
find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' |
|
||||
xargs grep -P '^(<<<<<<<|=======|>>>>>>>)$' | grep -P '.' && echo "Conflict markers are found in files"
|
||||
|
||||
# DOS/Windows newlines
|
||||
find $ROOT_PATH/{base,src,programs,utils,docs} -name '*.md' -or -name '*.h' -or -name '*.cpp' -or -name '*.js' -or -name '*.py' -or -name '*.html' | xargs grep -l -P '\r$' && echo "^ Files contain DOS/Windows newlines (\r\n instead of \n)."
|
||||
|
||||
# # workflows check
|
||||
# act --list --directory="$ROOT_PATH" 1>/dev/null 2>&1 || act --list --directory="$ROOT_PATH" 2>&1
|
||||
# actionlint -ignore 'reusable workflow call.+' || :
|
||||
37
ci_v2/jobs/scripts/check_style/double_whitespaces.pl
Executable file
37
ci_v2/jobs/scripts/check_style/double_whitespaces.pl
Executable file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
|
||||
# Find double whitespace such as "a, b, c" that looks very ugly and annoying.
|
||||
# But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines.
|
||||
|
||||
my $ret = 0;
|
||||
|
||||
foreach my $file (@ARGV)
|
||||
{
|
||||
my @array;
|
||||
|
||||
open (FH,'<',$file);
|
||||
while (<FH>)
|
||||
{
|
||||
push @array, $_;
|
||||
}
|
||||
|
||||
for (my $i = 1; $i < $#array; ++$i)
|
||||
{
|
||||
if ($array[$i] =~ ',( {2,3})[^ /]')
|
||||
{
|
||||
# https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
|
||||
|
||||
if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment
|
||||
&& (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/)
|
||||
&& $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1, 2 }
|
||||
{
|
||||
print($file . ":" . ($i + 1) . $array[$i]);
|
||||
$ret = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit $ret;
|
||||
251
ci_v2/settings/definitions.py
Normal file
251
ci_v2/settings/definitions.py
Normal file
@ -0,0 +1,251 @@
|
||||
from praktika import Docker, Secret
|
||||
|
||||
S3_BUCKET_NAME = "clickhouse-builds"
|
||||
S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com"
|
||||
|
||||
|
||||
class RunnerLabels:
|
||||
CI_SERVICES = "ci_services"
|
||||
CI_SERVICES_EBS = "ci_services_ebs"
|
||||
|
||||
|
||||
BASE_BRANCH = "master"
|
||||
|
||||
SECRETS = [
|
||||
Secret.Config(
|
||||
name="dockerhub_robot_password",
|
||||
type=Secret.Type.AWS_SSM_VAR,
|
||||
),
|
||||
Secret.Config(
|
||||
name="woolenwolf_gh_app.clickhouse-app-id",
|
||||
type=Secret.Type.AWS_SSM_SECRET,
|
||||
),
|
||||
Secret.Config(
|
||||
name="woolenwolf_gh_app.clickhouse-app-key",
|
||||
type=Secret.Type.AWS_SSM_SECRET,
|
||||
),
|
||||
]
|
||||
|
||||
DOCKERS = [
|
||||
# Docker.Config(
|
||||
# name="clickhouse/binary-builder",
|
||||
# path="./docker/packager/binary-builder",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/cctools",
|
||||
# path="./docker/packager/cctools",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/test-old-centos",
|
||||
# path="./docker/test/compatibility/centos",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/test-old-ubuntu",
|
||||
# path="./docker/test/compatibility/ubuntu",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/test-util",
|
||||
# path="./docker/test/util",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/integration-test",
|
||||
# path="./docker/test/integration/base",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/fuzzer",
|
||||
# path="./docker/test/fuzzer",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/performance-comparison",
|
||||
# path="./docker/test/performance-comparison",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=[],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/fasttest",
|
||||
# path="./docker/test/fasttest",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-util"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/test-base",
|
||||
# path="./docker/test/base",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-util"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/clickbench",
|
||||
# path="./docker/test/clickbench",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/keeper-jepsen-test",
|
||||
# path="./docker/test/keeper-jepsen",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/server-jepsen-test",
|
||||
# path="./docker/test/server-jepsen",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/sqllogic-test",
|
||||
# path="./docker/test/sqllogic",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/sqltest",
|
||||
# path="./docker/test/sqltest",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stateless-test",
|
||||
# path="./docker/test/stateless",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stateful-test",
|
||||
# path="./docker/test/stateful",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/stateless-test"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stress-test",
|
||||
# path="./docker/test/stress",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/stateful-test"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/unit-test",
|
||||
# path="./docker/test/unit",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/integration-tests-runner",
|
||||
# path="./docker/test/integration/runner",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
Docker.Config(
|
||||
name="clickhouse/style-test",
|
||||
path="./ci_v2/docker/style-test",
|
||||
platforms=Docker.Platforms.arm_amd,
|
||||
depends_on=[],
|
||||
),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/docs-builder",
|
||||
# path="./docker/docs/builder",
|
||||
# arm64=True,
|
||||
# amd64=True,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
]
|
||||
|
||||
# TODO:
|
||||
# "docker/test/integration/s3_proxy": {
|
||||
# "name": "clickhouse/s3-proxy",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/resolver": {
|
||||
# "name": "clickhouse/python-bottle",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/helper_container": {
|
||||
# "name": "clickhouse/integration-helper",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/mysql_golang_client": {
|
||||
# "name": "clickhouse/mysql-golang-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/dotnet_client": {
|
||||
# "name": "clickhouse/dotnet-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/mysql_java_client": {
|
||||
# "name": "clickhouse/mysql-java-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/mysql_js_client": {
|
||||
# "name": "clickhouse/mysql-js-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/mysql_php_client": {
|
||||
# "name": "clickhouse/mysql-php-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/postgresql_java_client": {
|
||||
# "name": "clickhouse/postgresql-java-client",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/kerberos_kdc": {
|
||||
# "only_amd64": true,
|
||||
# "name": "clickhouse/kerberos-kdc",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/kerberized_hadoop": {
|
||||
# "only_amd64": true,
|
||||
# "name": "clickhouse/kerberized-hadoop",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/sqlancer": {
|
||||
# "name": "clickhouse/sqlancer-test",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/install/deb": {
|
||||
# "name": "clickhouse/install-deb-test",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/install/rpm": {
|
||||
# "name": "clickhouse/install-rpm-test",
|
||||
# "dependent": []
|
||||
# },
|
||||
# "docker/test/integration/nginx_dav": {
|
||||
# "name": "clickhouse/nginx-dav",
|
||||
# "dependent": []
|
||||
# }
|
||||
|
||||
|
||||
class JobNames:
|
||||
STYLE_CHECK = "Style Check"
|
||||
20
ci_v2/settings/settings.py
Normal file
20
ci_v2/settings/settings.py
Normal file
@ -0,0 +1,20 @@
|
||||
from ci_v2.settings.definitions import (
|
||||
S3_BUCKET_HTTP_ENDPOINT,
|
||||
S3_BUCKET_NAME,
|
||||
RunnerLabels,
|
||||
)
|
||||
|
||||
S3_ARTIFACT_PATH = f"{S3_BUCKET_NAME}/artifacts"
|
||||
CI_CONFIG_RUNS_ON = [RunnerLabels.CI_SERVICES]
|
||||
DOCKER_BUILD_RUNS_ON = [RunnerLabels.CI_SERVICES_EBS]
|
||||
CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache"
|
||||
HTML_S3_PATH = f"{S3_BUCKET_NAME}/reports"
|
||||
S3_BUCKET_TO_HTTP_ENDPOINT = {S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT}
|
||||
|
||||
DOCKERHUB_USERNAME = "robotclickhouse"
|
||||
DOCKERHUB_SECRET = "dockerhub_robot_password"
|
||||
|
||||
CI_DB_DB_NAME = "default"
|
||||
CI_DB_TABLE_NAME = "checks"
|
||||
|
||||
INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = ""
|
||||
44
ci_v2/workflows/pull_request.py
Normal file
44
ci_v2/workflows/pull_request.py
Normal file
@ -0,0 +1,44 @@
|
||||
from typing import List
|
||||
|
||||
from ci_v2.settings.definitions import (
|
||||
BASE_BRANCH,
|
||||
DOCKERS,
|
||||
SECRETS,
|
||||
JobNames,
|
||||
RunnerLabels,
|
||||
)
|
||||
from praktika import Job, Workflow
|
||||
|
||||
style_check_job = Job.Config(
|
||||
name=JobNames.STYLE_CHECK,
|
||||
runs_on=[RunnerLabels.CI_SERVICES],
|
||||
command="python3 ./ci_v2/jobs/check_style.py",
|
||||
run_in_docker="clickhouse/style-test",
|
||||
)
|
||||
|
||||
workflow = Workflow.Config(
|
||||
name="PR",
|
||||
event=Workflow.Event.PULL_REQUEST,
|
||||
base_branches=[BASE_BRANCH],
|
||||
jobs=[
|
||||
style_check_job,
|
||||
],
|
||||
dockers=DOCKERS,
|
||||
secrets=SECRETS,
|
||||
enable_cache=True,
|
||||
enable_report=True,
|
||||
enable_merge_ready_status=True,
|
||||
)
|
||||
|
||||
WORKFLOWS = [
|
||||
workflow,
|
||||
] # type: List[Workflow.Config]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# example: local job test inside praktika environment
|
||||
from praktika.runner import Runner
|
||||
|
||||
Runner.generate_dummy_environment(workflow, style_check_job)
|
||||
|
||||
Runner().run(workflow, style_check_job)
|
||||
@ -48,6 +48,8 @@ if (NOT LINKER_NAME)
|
||||
find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
|
||||
elseif (OS_DARWIN)
|
||||
find_program (LLD_PATH NAMES "ld")
|
||||
# Duplicate libraries passed to the linker is not a problem.
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_warn_duplicate_libraries")
|
||||
endif ()
|
||||
if (LLD_PATH)
|
||||
if (OS_LINUX OR OS_DARWIN)
|
||||
|
||||
@ -1 +1,4 @@
|
||||
# See contrib/usearch-cmake/CMakeLists.txt
|
||||
set (FP16_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/FP16/")
|
||||
|
||||
add_library(_fp16 INTERFACE)
|
||||
target_include_directories(_fp16 SYSTEM INTERFACE ${FP16_PROJECT_DIR}/include)
|
||||
|
||||
2
contrib/SimSIMD
vendored
2
contrib/SimSIMD
vendored
@ -1 +1 @@
|
||||
Subproject commit 91a76d1ac519b3b9dc8957734a3dabd985f00c26
|
||||
Subproject commit ff51434d90c66f916e94ff05b24530b127aa4cff
|
||||
@ -1 +1,4 @@
|
||||
# See contrib/usearch-cmake/CMakeLists.txt
|
||||
set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
|
||||
|
||||
add_library(_simsimd INTERFACE)
|
||||
target_include_directories(_simsimd SYSTEM INTERFACE "${SIMSIMD_PROJECT_DIR}/include")
|
||||
|
||||
@ -69,6 +69,11 @@ set (SRCS_PROGRAM_OPTIONS
|
||||
"${LIBRARY_DIR}/libs/program_options/src/winmain.cpp"
|
||||
)
|
||||
|
||||
# Always compile this file with the highest possible level of optimizations, even in Debug builds.
|
||||
# Otherwise the client takes too long to start and SQL stateless tests (many options to parse)
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/65745
|
||||
set_source_files_properties(${SRCS_PROGRAM_OPTIONS} PROPERTIES COMPILE_FLAGS "-O3")
|
||||
|
||||
add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS})
|
||||
add_library (boost::program_options ALIAS _boost_program_options)
|
||||
target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})
|
||||
|
||||
2
contrib/grpc
vendored
2
contrib/grpc
vendored
@ -1 +1 @@
|
||||
Subproject commit 7bc3abe952aba1dc7bce7f2f790dc781cb51a41e
|
||||
Subproject commit 62e871c36fa93c0af939bd31762845265214fe3d
|
||||
@ -164,15 +164,6 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
# Because our coverage callbacks call malloc, and recursive call of malloc could not work.
|
||||
target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST})
|
||||
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||
target_compile_definitions(_jemalloc PRIVATE
|
||||
-DJEMALLOC_DEBUG=1
|
||||
# Usage examples:
|
||||
# - MALLOC_CONF=log:.
|
||||
# - MALLOC_CONF='log:core.malloc.exit|core.sallocx.entry|core.sdallocx.entry'
|
||||
-DJEMALLOC_LOG=1)
|
||||
endif ()
|
||||
|
||||
target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1)
|
||||
|
||||
# jemalloc provides support two unwind flavors:
|
||||
|
||||
2
contrib/libdivide
vendored
2
contrib/libdivide
vendored
@ -1 +1 @@
|
||||
Subproject commit 3bd34388573681ce563348cdf04fe15d24770d04
|
||||
Subproject commit 01526031eb79375dc85e0212c966d2c514a01234
|
||||
@ -50,7 +50,11 @@ if(NOT MARIADB_UNIX_ADDR)
|
||||
set(MARIADB_UNIX_ADDR "/tmp/mysql.sock")
|
||||
endif()
|
||||
|
||||
set(HAVE_ALLOCA_H 1)
|
||||
if (OS_FREEBSD)
|
||||
set(HAVE_ALLOCA_H 0)
|
||||
else()
|
||||
set(HAVE_ALLOCA_H 1)
|
||||
endif()
|
||||
set(HAVE_ARPA_INET_H 1)
|
||||
set(HAVE_DLFCN_H 1)
|
||||
set(HAVE_FCNTL_H 1)
|
||||
|
||||
2
contrib/simdjson
vendored
2
contrib/simdjson
vendored
@ -1 +1 @@
|
||||
Subproject commit 6060be2fdf62edf4a8f51a8b0883d57d09397b30
|
||||
Subproject commit e341c8b43861b43de29c48ab65f292d997096953
|
||||
2
contrib/usearch
vendored
2
contrib/usearch
vendored
@ -1 +1 @@
|
||||
Subproject commit 7a8967cb442b08ca20c3dd781414378e65957d37
|
||||
Subproject commit d1d33eac94acd3b628e0b446c927ec3295ef63c7
|
||||
@ -1,14 +1,9 @@
|
||||
set(FP16_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/FP16")
|
||||
set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD")
|
||||
set(USEARCH_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/usearch")
|
||||
|
||||
add_library(_usearch INTERFACE)
|
||||
target_include_directories(_usearch SYSTEM INTERFACE ${USEARCH_PROJECT_DIR}/include)
|
||||
|
||||
target_include_directories(_usearch SYSTEM INTERFACE
|
||||
${FP16_PROJECT_DIR}/include
|
||||
${SIMSIMD_PROJECT_DIR}/include
|
||||
${USEARCH_PROJECT_DIR}/include)
|
||||
|
||||
target_link_libraries(_usearch INTERFACE _fp16)
|
||||
target_compile_definitions(_usearch INTERFACE USEARCH_USE_FP16LIB)
|
||||
|
||||
# target_compile_definitions(_usearch INTERFACE USEARCH_USE_SIMSIMD)
|
||||
|
||||
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -2,19 +2,29 @@
|
||||
# To run this script you must install docker and piddeptree python package
|
||||
#
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def build_docker_deps(image_name, imagedir):
|
||||
cmd = f"""docker run --entrypoint "/bin/bash" {image_name} -c "pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze --warn silence | sed 's/ \+//g' | sort | uniq" > {imagedir}/requirements.txt"""
|
||||
def build_docker_deps(image_name: str, imagedir: str) -> None:
|
||||
print("Fetch the newest manifest for", image_name)
|
||||
pip_cmd = (
|
||||
"pip install pipdeptree 2>/dev/null 1>/dev/null && pipdeptree --freeze "
|
||||
"--warn silence --exclude pipdeptree"
|
||||
)
|
||||
# /=/!d - remove dependencies without pin
|
||||
# ubuntu - ignore system packages
|
||||
# \s - remove spaces
|
||||
sed = r"sed '/==/!d; /==.*+ubuntu/d; s/\s//g'"
|
||||
cmd = rf"""docker run --rm --entrypoint "/bin/bash" {image_name} -c "{pip_cmd} | {sed} | sort -u" > {imagedir}/requirements.txt"""
|
||||
print("Running the command:", cmd)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
def check_docker_file_install_with_pip(filepath):
|
||||
image_name = None
|
||||
with open(filepath, "r") as f:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if "docker build" in line:
|
||||
arr = line.split(" ")
|
||||
@ -25,7 +35,7 @@ def check_docker_file_install_with_pip(filepath):
|
||||
return image_name, False
|
||||
|
||||
|
||||
def process_affected_images(images_dir):
|
||||
def process_affected_images(images_dir: str) -> None:
|
||||
for root, _dirs, files in os.walk(images_dir):
|
||||
for f in files:
|
||||
if f == "Dockerfile":
|
||||
|
||||
@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.8.4.13"
|
||||
ARG VERSION="24.9.2.42"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from argparse import ArgumentParser
|
||||
import os
|
||||
import jinja2
|
||||
import itertools
|
||||
import os
|
||||
from argparse import ArgumentParser
|
||||
|
||||
import jinja2
|
||||
|
||||
|
||||
def removesuffix(text, suffix):
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import datetime
|
||||
import os
|
||||
import subprocess
|
||||
import datetime
|
||||
from flask import Flask, flash, request, redirect, url_for
|
||||
|
||||
from flask import Flask, flash, redirect, request, url_for
|
||||
|
||||
|
||||
def run_command(command, wait=False):
|
||||
|
||||
@ -48,7 +48,7 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
|
||||
&& add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
|
||||
&& apt-get update \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
docker-ce='5:23.*' \
|
||||
docker-ce='5:23.*' docker-compose-plugin='2.29.*' \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
|
||||
@ -0,0 +1 @@
|
||||
[rabbitmq_consistent_hash_exchange].
|
||||
@ -13,3 +13,5 @@ ssl_options.fail_if_no_peer_cert = false
|
||||
ssl_options.cacertfile = /etc/rabbitmq/ca-cert.pem
|
||||
ssl_options.certfile = /etc/rabbitmq/server-cert.pem
|
||||
ssl_options.keyfile = /etc/rabbitmq/server-key.pem
|
||||
|
||||
vm_memory_high_watermark.absolute = 2GB
|
||||
|
||||
@ -1,15 +1,13 @@
|
||||
PyHDFS==0.3.1
|
||||
PyJWT==2.3.0
|
||||
PyMySQL==1.1.0
|
||||
PyJWT==2.4.0
|
||||
PyMySQL==1.1.1
|
||||
PyNaCl==1.5.0
|
||||
PyYAML==5.3.1
|
||||
SecretStorage==3.3.1
|
||||
argon2-cffi-bindings==21.2.0
|
||||
argon2-cffi==23.1.0
|
||||
async-timeout==4.0.3
|
||||
asyncio==3.4.3
|
||||
attrs==23.2.0
|
||||
avro==1.10.2
|
||||
avro==1.11.3
|
||||
azure-core==1.30.1
|
||||
azure-storage-blob==12.19.0
|
||||
bcrypt==4.1.3
|
||||
@ -24,18 +22,13 @@ cffi==1.16.0
|
||||
charset-normalizer==3.3.2
|
||||
click==8.1.7
|
||||
confluent-kafka==2.3.0
|
||||
cryptography==3.4.8
|
||||
cryptography==42.0.0
|
||||
dbus-python==1.2.18
|
||||
decorator==5.1.1
|
||||
delta-spark==2.3.0
|
||||
deltalake==0.16.0
|
||||
dict2xml==1.7.4
|
||||
dicttoxml==1.7.16
|
||||
distro-info==1.1+ubuntu0.2
|
||||
distro==1.7.0
|
||||
docker-compose==1.29.2
|
||||
docker==6.1.3
|
||||
dockerpty==0.4.1
|
||||
docopt==0.6.2
|
||||
exceptiongroup==1.2.1
|
||||
execnet==2.1.1
|
||||
geomet==0.2.1.post1
|
||||
@ -49,7 +42,6 @@ iniconfig==2.0.0
|
||||
isodate==0.6.1
|
||||
jeepney==0.7.1
|
||||
jmespath==1.0.1
|
||||
jsonschema==3.2.0
|
||||
jwcrypto==1.5.6
|
||||
kafka-python==2.0.2
|
||||
kazoo==2.9.0
|
||||
@ -63,23 +55,22 @@ lz4==4.3.3
|
||||
minio==7.2.3
|
||||
more-itertools==8.10.0
|
||||
nats-py==2.6.0
|
||||
numpy==2.1.0
|
||||
oauthlib==3.2.0
|
||||
packaging==24.0
|
||||
paramiko==3.4.0
|
||||
pika==1.2.0
|
||||
pip==24.1.1
|
||||
pipdeptree==2.23.0
|
||||
pluggy==1.5.0
|
||||
protobuf==4.25.2
|
||||
psycopg2-binary==2.9.6
|
||||
py4j==0.10.9.5
|
||||
py==1.11.0
|
||||
pyarrow-hotfix==0.6
|
||||
pyarrow==17.0.0
|
||||
pycparser==2.22
|
||||
pycryptodome==3.20.0
|
||||
pymongo==3.11.0
|
||||
pyparsing==2.4.7
|
||||
pyrsistent==0.20.0
|
||||
pyspark==3.3.2
|
||||
pyspnego==0.10.2
|
||||
pytest-order==1.0.0
|
||||
@ -89,28 +80,22 @@ pytest-reportlog==0.4.0
|
||||
pytest-timeout==2.2.0
|
||||
pytest-xdist==3.5.0
|
||||
pytest==7.4.4
|
||||
python-apt==2.4.0+ubuntu3
|
||||
python-dateutil==2.9.0.post0
|
||||
python-dotenv==0.21.1
|
||||
pytz==2023.3.post1
|
||||
redis==5.0.1
|
||||
requests-kerberos==0.14.0
|
||||
requests==2.31.0
|
||||
retry==0.9.2
|
||||
s3transfer==0.10.1
|
||||
setuptools==59.6.0
|
||||
setuptools==70.0.0
|
||||
simplejson==3.19.2
|
||||
six==1.16.0
|
||||
soupsieve==2.5
|
||||
texttable==1.7.0
|
||||
tomli==2.0.1
|
||||
typing_extensions==4.11.0
|
||||
tzlocal==2.1
|
||||
unattended-upgrades==0.1
|
||||
urllib3==2.0.7
|
||||
wadllib==1.3.6
|
||||
websocket-client==0.59.0
|
||||
wheel==0.37.1
|
||||
websocket-client==1.8.0
|
||||
wheel==0.38.1
|
||||
zipp==1.0.0
|
||||
deltalake==0.16.0
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import csv
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import yaml
|
||||
import html
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
from clickhouse_driver import Client
|
||||
|
||||
import yaml
|
||||
from clickhouse_driver import Client
|
||||
|
||||
client = Client(host="localhost", port=9000)
|
||||
settings = {
|
||||
|
||||
@ -41,7 +41,7 @@ sidebar_label: 2022
|
||||
* Backported in [#25364](https://github.com/ClickHouse/ClickHouse/issues/25364): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#25387](https://github.com/ClickHouse/ClickHouse/issues/25387): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#25455](https://github.com/ClickHouse/ClickHouse/issues/25455): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#25406](https://github.com/ClickHouse/ClickHouse/issues/25406): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#25406](https://github.com/ClickHouse/ClickHouse/issues/25406): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formatted query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#25505](https://github.com/ClickHouse/ClickHouse/issues/25505): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
@ -40,7 +40,7 @@ sidebar_label: 2022
|
||||
* Backported in [#25362](https://github.com/ClickHouse/ClickHouse/issues/25362): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#25386](https://github.com/ClickHouse/ClickHouse/issues/25386): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#25456](https://github.com/ClickHouse/ClickHouse/issues/25456): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#25408](https://github.com/ClickHouse/ClickHouse/issues/25408): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#25408](https://github.com/ClickHouse/ClickHouse/issues/25408): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formatted query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#25504](https://github.com/ClickHouse/ClickHouse/issues/25504): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
@ -24,7 +24,7 @@ sidebar_label: 2022
|
||||
* Backported in [#25363](https://github.com/ClickHouse/ClickHouse/issues/25363): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#25388](https://github.com/ClickHouse/ClickHouse/issues/25388): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#25448](https://github.com/ClickHouse/ClickHouse/issues/25448): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formatted query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
|
||||
@ -133,7 +133,7 @@ sidebar_label: 2022
|
||||
* On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix `REPLACE` column transformer when used in DDL by correctly quoting the formatted query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix excessive underscore before the names of the preprocessed configuration files. [#25431](https://github.com/ClickHouse/ClickHouse/pull/25431) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix convertion of datetime with timezone for MySQL, PostgreSQL, ODBC. Closes [#5057](https://github.com/ClickHouse/ClickHouse/issues/5057). [#25528](https://github.com/ClickHouse/ClickHouse/pull/25528) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
49
docs/changelogs/v24.3.12.75-lts.md
Normal file
49
docs/changelogs/v24.3.12.75-lts.md
Normal file
@ -0,0 +1,49 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.3.12.75-lts (7cb5dff8019) FIXME as compared to v24.3.11.7-lts (28795d0a47e)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#69607](https://github.com/ClickHouse/ClickHouse/issues/69607): Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#69785](https://github.com/ClickHouse/ClickHouse/issues/69785): Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
|
||||
* Backported in [#69461](https://github.com/ClickHouse/ClickHouse/issues/69461): Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#68217](https://github.com/ClickHouse/ClickHouse/issues/68217): Fixed a NULL pointer dereference, triggered by a specially crafted query, that crashed the server via hopEnd, hopStart, tumbleEnd, and tumbleStart. [#68098](https://github.com/ClickHouse/ClickHouse/pull/68098) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Backported in [#69437](https://github.com/ClickHouse/ClickHouse/issues/69437): After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
|
||||
* Backported in [#69827](https://github.com/ClickHouse/ClickHouse/issues/69827): Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69294](https://github.com/ClickHouse/ClickHouse/issues/69294): Fix merging of aggregated data for grouping sets. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#70470](https://github.com/ClickHouse/ClickHouse/issues/70470): Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Backported in [#69456](https://github.com/ClickHouse/ClickHouse/issues/69456): Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#69497](https://github.com/ClickHouse/ClickHouse/issues/69497): Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#69724](https://github.com/ClickHouse/ClickHouse/issues/69724): Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#69668](https://github.com/ClickHouse/ClickHouse/issues/69668): Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#69792](https://github.com/ClickHouse/ClickHouse/issues/69792): Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#70089](https://github.com/ClickHouse/ClickHouse/issues/70089): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#70077](https://github.com/ClickHouse/ClickHouse/issues/70077): Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#70162](https://github.com/ClickHouse/ClickHouse/issues/70162): Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#70232](https://github.com/ClickHouse/ClickHouse/issues/70232): Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70179](https://github.com/ClickHouse/ClickHouse/issues/70179): Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70241](https://github.com/ClickHouse/ClickHouse/issues/70241): Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Backported in [#70397](https://github.com/ClickHouse/ClickHouse/issues/70397): Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#69526](https://github.com/ClickHouse/ClickHouse/issues/69526):. [#67029](https://github.com/ClickHouse/ClickHouse/pull/67029) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#69506](https://github.com/ClickHouse/ClickHouse/issues/69506): Better handling of errors from azure storage. [#62306](https://github.com/ClickHouse/ClickHouse/pull/62306) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#69955](https://github.com/ClickHouse/ClickHouse/issues/69955): Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Backported in [#69485](https://github.com/ClickHouse/ClickHouse/issues/69485): Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Backported in [#70028](https://github.com/ClickHouse/ClickHouse/issues/70028): Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Backported in [#69421](https://github.com/ClickHouse/ClickHouse/issues/69421): Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#69747](https://github.com/ClickHouse/ClickHouse/issues/69747): Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#69636](https://github.com/ClickHouse/ClickHouse/issues/69636): Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* The test is flaky and the feature experimental. [#70269](https://github.com/ClickHouse/ClickHouse/pull/70269) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix test distributed inter server secret in 24.3. [#70325](https://github.com/ClickHouse/ClickHouse/pull/70325) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
32
docs/changelogs/v24.7.5.37-stable.md
Normal file
32
docs/changelogs/v24.7.5.37-stable.md
Normal file
@ -0,0 +1,32 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.7.5.37-stable (f2533ca97be) FIXME as compared to v24.7.4.51-stable (70fe2f6fa52)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#68971](https://github.com/ClickHouse/ClickHouse/issues/68971): Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
|
||||
* Backported in [#68816](https://github.com/ClickHouse/ClickHouse/issues/68816): Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#69006](https://github.com/ClickHouse/ClickHouse/issues/69006): After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68884](https://github.com/ClickHouse/ClickHouse/issues/68884): Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Backported in [#69027](https://github.com/ClickHouse/ClickHouse/issues/69027): Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#68862](https://github.com/ClickHouse/ClickHouse/issues/68862): Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#68788](https://github.com/ClickHouse/ClickHouse/issues/68788): Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Backported in [#68909](https://github.com/ClickHouse/ClickHouse/issues/68909): Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#69158](https://github.com/ClickHouse/ClickHouse/issues/69158): Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69118](https://github.com/ClickHouse/ClickHouse/issues/69118): Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#68944](https://github.com/ClickHouse/ClickHouse/issues/68944):. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#68832](https://github.com/ClickHouse/ClickHouse/issues/68832): Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#68779](https://github.com/ClickHouse/ClickHouse/issues/68779): Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#68960](https://github.com/ClickHouse/ClickHouse/issues/68960): Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([jsc0218](https://github.com/jsc0218)).
|
||||
* Backported in [#69050](https://github.com/ClickHouse/ClickHouse/issues/69050): Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
73
docs/changelogs/v24.8.5.115-lts.md
Normal file
73
docs/changelogs/v24.8.5.115-lts.md
Normal file
@ -0,0 +1,73 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.8.5.115-lts (8c4cb00a384) FIXME as compared to v24.8.4.13-lts (53195bc189b)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#70046](https://github.com/ClickHouse/ClickHouse/issues/70046): Add new column readonly_duration to the system.replicas table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#69786](https://github.com/ClickHouse/ClickHouse/issues/69786): Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
|
||||
* Backported in [#70318](https://github.com/ClickHouse/ClickHouse/issues/70318): Fixed error on generated columns in MaterializedPostgreSQL when adnum ordering is broken [#63161](https://github.com/ClickHouse/ClickHouse/issues/63161). Fixed error on id column with nextval expression as default MaterializedPostgreSQL when there are generated columns in table. Fixed error on dropping publication with symbols except [a-z1-9-]. [#67664](https://github.com/ClickHouse/ClickHouse/pull/67664) ([Kruglov Kirill](https://github.com/1on)).
|
||||
* Backported in [#69467](https://github.com/ClickHouse/ClickHouse/issues/69467): Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#69735](https://github.com/ClickHouse/ClickHouse/issues/69735): Fix crash in `lag`/`lead` which is introduced in [#67091](https://github.com/ClickHouse/ClickHouse/issues/67091). [#68262](https://github.com/ClickHouse/ClickHouse/pull/68262) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Backported in [#69444](https://github.com/ClickHouse/ClickHouse/issues/69444): After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
|
||||
* Backported in [#69810](https://github.com/ClickHouse/ClickHouse/issues/69810): Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69757](https://github.com/ClickHouse/ClickHouse/issues/69757): Fix incorrect results of Fix uniq and GROUP BY for JSON/Dynamic types. [#69203](https://github.com/ClickHouse/ClickHouse/pull/69203) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70195](https://github.com/ClickHouse/ClickHouse/issues/70195): Fix insertion of incomplete type into Dynamic during deserialization. It could lead to `Parameter out of bound` errors. [#69291](https://github.com/ClickHouse/ClickHouse/pull/69291) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#69398](https://github.com/ClickHouse/ClickHouse/issues/69398): Mark Dynamic type as not safe primary key type to avoid issues with Fields. [#69311](https://github.com/ClickHouse/ClickHouse/pull/69311) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#69704](https://github.com/ClickHouse/ClickHouse/issues/69704): Improve restoring of access entities' dependencies. [#69346](https://github.com/ClickHouse/ClickHouse/pull/69346) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#69459](https://github.com/ClickHouse/ClickHouse/issues/69459): Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#69503](https://github.com/ClickHouse/ClickHouse/issues/69503): Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#69480](https://github.com/ClickHouse/ClickHouse/issues/69480): Quick fix for s3queue problem on 24.6 or create query with database replicated. [#69454](https://github.com/ClickHouse/ClickHouse/pull/69454) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#69535](https://github.com/ClickHouse/ClickHouse/issues/69535): Fixed case when memory consumption was too high because of the squashing in `INSERT INTO ... SELECT` or `CREATE TABLE AS SELECT` queries. [#69469](https://github.com/ClickHouse/ClickHouse/pull/69469) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Backported in [#69696](https://github.com/ClickHouse/ClickHouse/issues/69696): Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70439](https://github.com/ClickHouse/ClickHouse/issues/70439): Fix vrash during insertion into FixedString column in PostgreSQL engine. [#69584](https://github.com/ClickHouse/ClickHouse/pull/69584) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#69666](https://github.com/ClickHouse/ClickHouse/issues/69666): Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#70191](https://github.com/ClickHouse/ClickHouse/issues/70191): Fix crash when executing `create view t as (with recursive 42 as ttt select ttt);`. [#69676](https://github.com/ClickHouse/ClickHouse/pull/69676) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Backported in [#69798](https://github.com/ClickHouse/ClickHouse/issues/69798): Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#70410](https://github.com/ClickHouse/ClickHouse/issues/70410): Fixed `maxMapState` throwing 'Bad get' if value type is DateTime64. [#69787](https://github.com/ClickHouse/ClickHouse/pull/69787) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#70019](https://github.com/ClickHouse/ClickHouse/issues/70019): Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69941](https://github.com/ClickHouse/ClickHouse/issues/69941): Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70001](https://github.com/ClickHouse/ClickHouse/issues/70001): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#70081](https://github.com/ClickHouse/ClickHouse/issues/70081): Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#70068](https://github.com/ClickHouse/ClickHouse/issues/70068): Fixes `Block structure mismatch` for queries with nested views and `WHERE` condition. Fixes [#66209](https://github.com/ClickHouse/ClickHouse/issues/66209). [#70054](https://github.com/ClickHouse/ClickHouse/pull/70054) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#70166](https://github.com/ClickHouse/ClickHouse/issues/70166): Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#70236](https://github.com/ClickHouse/ClickHouse/issues/70236): Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70203](https://github.com/ClickHouse/ClickHouse/issues/70203): Fix wrong result with skipping index. [#70127](https://github.com/ClickHouse/ClickHouse/pull/70127) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#70183](https://github.com/ClickHouse/ClickHouse/issues/70183): Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70251](https://github.com/ClickHouse/ClickHouse/issues/70251): Fix possible hung in ALTER COLUMN with Dynamic type. [#70144](https://github.com/ClickHouse/ClickHouse/pull/70144) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70228](https://github.com/ClickHouse/ClickHouse/issues/70228): Use correct `max_types` parameter during Dynamic type creation for JSON subcolumn. [#70147](https://github.com/ClickHouse/ClickHouse/pull/70147) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70243](https://github.com/ClickHouse/ClickHouse/issues/70243): Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Backported in [#70432](https://github.com/ClickHouse/ClickHouse/issues/70432): Fix possible crash in JSON column. [#70172](https://github.com/ClickHouse/ClickHouse/pull/70172) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70307](https://github.com/ClickHouse/ClickHouse/issues/70307): Fix multiple issues with arrayMin and arrayMax. [#70207](https://github.com/ClickHouse/ClickHouse/pull/70207) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#70274](https://github.com/ClickHouse/ClickHouse/issues/70274): Respect setting allow_simdjson in JSON type parser. [#70218](https://github.com/ClickHouse/ClickHouse/pull/70218) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70345](https://github.com/ClickHouse/ClickHouse/issues/70345): Don't modify global settings with startup scripts. Previously, changing a setting in a startup script would change it globally. [#70310](https://github.com/ClickHouse/ClickHouse/pull/70310) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#70426](https://github.com/ClickHouse/ClickHouse/issues/70426): Fix ALTER of Dynamic type with reducing max_types parameter that could lead to server crash. [#70328](https://github.com/ClickHouse/ClickHouse/pull/70328) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70371](https://github.com/ClickHouse/ClickHouse/issues/70371): Fix crash when using WITH FILL incorrectly. [#70338](https://github.com/ClickHouse/ClickHouse/pull/70338) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
* NO CL ENTRY: 'Revert "Backport [#70146](https://github.com/ClickHouse/ClickHouse/issues/70146) to 24.8: Upgrade integration-runner image"'. [#70324](https://github.com/ClickHouse/ClickHouse/pull/70324) ([Max K.](https://github.com/maxknv)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#69961](https://github.com/ClickHouse/ClickHouse/issues/69961): Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Backported in [#69491](https://github.com/ClickHouse/ClickHouse/issues/69491): Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Backported in [#69953](https://github.com/ClickHouse/ClickHouse/issues/69953): Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Backported in [#69353](https://github.com/ClickHouse/ClickHouse/issues/69353): Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#69427](https://github.com/ClickHouse/ClickHouse/issues/69427): Fix 24.8 setting compatibility `rows_before_aggregation`. [#69394](https://github.com/ClickHouse/ClickHouse/pull/69394) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Backported in [#69689](https://github.com/ClickHouse/ClickHouse/issues/69689): Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#69639](https://github.com/ClickHouse/ClickHouse/issues/69639): Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#69721](https://github.com/ClickHouse/ClickHouse/issues/69721): Prohibit `ALTER TABLE ... ADD INDEX ... TYPE` inverted if setting = 0. [#69684](https://github.com/ClickHouse/ClickHouse/pull/69684) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#69972](https://github.com/ClickHouse/ClickHouse/issues/69972): S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#70283](https://github.com/ClickHouse/ClickHouse/issues/70283): Improve pipdeptree generator for docker images. - Update requirements.txt for the integration tests runner container - Remove some small dependencies, improve `helpers/retry_decorator.py` - Upgrade docker-compose from EOL version 1 to version 2. [#70146](https://github.com/ClickHouse/ClickHouse/pull/70146) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#70260](https://github.com/ClickHouse/ClickHouse/issues/70260): Update test_storage_s3_queue/test.py. [#70159](https://github.com/ClickHouse/ClickHouse/pull/70159) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#70314](https://github.com/ClickHouse/ClickHouse/issues/70314): CI: Remove await feature from release branches. [#70294](https://github.com/ClickHouse/ClickHouse/pull/70294) ([Max K.](https://github.com/maxknv)).
|
||||
* Backported in [#70380](https://github.com/ClickHouse/ClickHouse/issues/70380): Fix tiny mistake, responsible for some of kafka test flaps. Example [report](https://s3.amazonaws.com/clickhouse-test-reports/0/3198aafac59c368993e7b5f49d95674cc1b1be18/integration_tests__release__[2_4].html). [#70352](https://github.com/ClickHouse/ClickHouse/pull/70352) ([filimonov](https://github.com/filimonov)).
|
||||
* Backported in [#70405](https://github.com/ClickHouse/ClickHouse/issues/70405): Closes [#69634](https://github.com/ClickHouse/ClickHouse/issues/69634). [#70354](https://github.com/ClickHouse/ClickHouse/pull/70354) ([pufit](https://github.com/pufit)).
|
||||
|
||||
500
docs/changelogs/v24.9.1.3278-stable.md
Normal file
500
docs/changelogs/v24.9.1.3278-stable.md
Normal file
@ -0,0 +1,500 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.9.1.3278-stable (6d058d82a8e) FIXME as compared to v24.9.1.1-new (e02b434d2fc)
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Allow to write `SETTINGS` before `FORMAT` in a chain of queries with `UNION` when subqueries are inside parentheses. This closes [#39712](https://github.com/ClickHouse/ClickHouse/issues/39712). Change the behavior when a query has the SETTINGS clause specified twice in a sequence. The closest SETTINGS clause will have a preference for the corresponding subquery. In the previous versions, the outermost SETTINGS clause could take a preference over the inner one. [#60197](https://github.com/ClickHouse/ClickHouse/pull/60197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Do not allow explicitly specifying UUID when creating a table in Replicated database. Also, do not allow explicitly specifying ZooKeeper path and replica name for *MergeTree tables in Replicated databases. [#66104](https://github.com/ClickHouse/ClickHouse/pull/66104) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Reimplement Dynamic type. Now when the limit of dynamic data types is reached new types are not casted to String but stored in a special data structure in binary format with binary encoded data type. Now any type ever inserted into Dynamic column can be read from it as subcolumn. [#68132](https://github.com/ClickHouse/ClickHouse/pull/68132) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Expressions like `a[b].c` are supported for named tuples, as well as named subscripts from arbitrary expressions, e.g., `expr().name`. This is useful for processing JSON. This closes [#54965](https://github.com/ClickHouse/ClickHouse/issues/54965). In previous versions, an expression of form `expr().name` was parsed as `tupleElement(expr(), name)`, and the query analyzer was searching for a column `name` rather than for the corresponding tuple element; while in the new version, it is changed to `tupleElement(expr(), 'name')`. In most cases, the previous version was not working, but it is possible to imagine a very unusual scenario when this change could lead to incompatibility: if you stored names of tuple elements in a column or an alias, that was named differently than the tuple element's name: `SELECT 'b' AS a, CAST([tuple(123)] AS 'Array(Tuple(b UInt8))') AS t, t[1].a`. It is very unlikely that you used such queries, but we still have to mark this change as potentially backward incompatible. [#68435](https://github.com/ClickHouse/ClickHouse/pull/68435) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* When the setting `print_pretty_type_names` is enabled, it will print `Tuple` data type in a pretty form in `SHOW CREATE TABLE` statements, `formatQuery` function, and in the interactive mode in `clickhouse-client` and `clickhouse-local`. In previous versions, this setting was only applied to `DESCRIBE` queries and `toTypeName`. This closes [#65753](https://github.com/ClickHouse/ClickHouse/issues/65753). [#68492](https://github.com/ClickHouse/ClickHouse/pull/68492) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### New Feature
|
||||
* Function `toStartOfInterval()` now has a new overload which emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function. ([#55619](https://github.com/ClickHouse/ClickHouse/issues/55619)). It allows to align date or timestamp values to multiples of a given interval from an *arbitrary* origin (instead of 0000-01-01 00:00:00.000 as *fixed* origin). For example, `SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));` returns `2023-01-01 14:44:30` which is a multiple of 1 minute intervals, starting from origin `2023-01-01 14:35:30`. [#56738](https://github.com/ClickHouse/ClickHouse/pull/56738) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Add support for `ATTACH PARTITION ALL FROM`. [#61987](https://github.com/ClickHouse/ClickHouse/pull/61987) ([Kirill Nikiforov](https://github.com/allmazz)).
|
||||
* Adds a setting `input_format_try_infer_variants` which allows Variant type to be inferred during schema inference for text formats when there is more than one possible type for column/array elements. [#63798](https://github.com/ClickHouse/ClickHouse/pull/63798) ([Shaun Struwig](https://github.com/Blargian)).
|
||||
* Introduced JSONCompactWithProgress format where ClickHouse outputs each row as a newline-delimited JSON object, including metadata, data, progress, totals, and statistics. [#66205](https://github.com/ClickHouse/ClickHouse/pull/66205) ([Alexey Korepanov](https://github.com/alexkorep)).
|
||||
* Implement new JSON data type. [#66444](https://github.com/ClickHouse/ClickHouse/pull/66444) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Add the `input_format_json_empty_as_default` setting which, when enabled, treats empty fields in JSON inputs as default values. Closes [#59339](https://github.com/ClickHouse/ClickHouse/issues/59339). [#66782](https://github.com/ClickHouse/ClickHouse/pull/66782) ([Alexis Arnaud](https://github.com/a-a-f)).
|
||||
* Added functions `overlay` and `overlayUTF8` which replace parts of a string by another string. Example: `SELECT overlay('Hello New York', 'Jersey', 11)` returns `Hello New Jersey`. [#66933](https://github.com/ClickHouse/ClickHouse/pull/66933) ([李扬](https://github.com/taiyang-li)).
|
||||
* Add new Command, Lightweight Delete In Partition ``` DELETE FROM [db.]table [ON CLUSTER cluster] [IN PARTITION partition_expr] WHERE expr; ``` ``` VM-114-29-tos :) select * from ads_app_poster_ip_source_channel_di_replicated_local;. [#67805](https://github.com/ClickHouse/ClickHouse/pull/67805) ([sunny](https://github.com/sunny19930321)).
|
||||
* Implemented comparison for `Interval` data type values so they are converting now to the least supertype. [#68057](https://github.com/ClickHouse/ClickHouse/pull/68057) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Add create_if_not_exists setting to default to IF NOT EXISTS behavior during CREATE statements. [#68164](https://github.com/ClickHouse/ClickHouse/pull/68164) ([Peter Nguyen](https://github.com/petern48)).
|
||||
* Makes possible to read Iceberg tables in Azure and locally. [#68210](https://github.com/ClickHouse/ClickHouse/pull/68210) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Add aggregate functions distinctDynamicTypes/distinctJSONPaths/distinctJSONPathsAndTypes for better introspection of JSON column type content. [#68463](https://github.com/ClickHouse/ClickHouse/pull/68463) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Query cache entries can now be dropped by tag. For example, the query cache entry created by `SELECT 1 SETTINGS use_query_cache = true, query_cache_tag = 'abc'` can now be dropped by `SYSTEM DROP QUERY CACHE TAG 'abc'` (or of course just: `SYSTEM DROP QUERY CACHE` which will clear the entire query cache). [#68477](https://github.com/ClickHouse/ClickHouse/pull/68477) ([Michał Tabaszewski](https://github.com/pinsvin00)).
|
||||
* Add storage encryption for named collections. [#68615](https://github.com/ClickHouse/ClickHouse/pull/68615) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Added `ripeMD160` function, which computes the RIPEMD-160 cryptographic hash of a string. Example: `SELECT hex(ripeMD160('The quick brown fox jumps over the lazy dog'))` returns `37F332F68DB77BD9D7EDD4969571AD671CF9DD3B`. [#68639](https://github.com/ClickHouse/ClickHouse/pull/68639) ([Dergousov Maxim](https://github.com/m7kss1)).
|
||||
* Add virtual column _headers for url table engine. Closes [#65026](https://github.com/ClickHouse/ClickHouse/issues/65026). [#68867](https://github.com/ClickHouse/ClickHouse/pull/68867) ([flynn](https://github.com/ucasfl)).
|
||||
* Adding `system.projections` table to track available projections. [#68901](https://github.com/ClickHouse/ClickHouse/pull/68901) ([Jordi Villar](https://github.com/jrdi)).
|
||||
* Add new function `arrayZipUnaligned` for spark compatiablity(arrays_zip), which allowed unaligned arrays based on original `arrayZip`. ``` sql SELECT arrayZipUnaligned([1], [1, 2, 3]). [#69030](https://github.com/ClickHouse/ClickHouse/pull/69030) ([李扬](https://github.com/taiyang-li)).
|
||||
* Adding `RealTimeMicroseconds` metric in the HTTP Header `X-ClickHouse-Summary`. This way we can know the CPU time of a request without having to go to check it in `system.query_log`. [#69032](https://github.com/ClickHouse/ClickHouse/pull/69032) ([Alejandro](https://github.com/alexon1234)).
|
||||
* Added cp/mv commands for keeper client which atomically copies/moves node. [#69034](https://github.com/ClickHouse/ClickHouse/pull/69034) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Adds argument `scale` (default: `true`) to function `arrayAUC` which allows to skip the normalization step (issue [#69609](https://github.com/ClickHouse/ClickHouse/issues/69609)). [#69717](https://github.com/ClickHouse/ClickHouse/pull/69717) ([gabrielmcg44](https://github.com/gabrielmcg44)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Improve the join performance by rerange the right table by keys while the table keys are dense in left or inner hash join. [#60341](https://github.com/ClickHouse/ClickHouse/pull/60341) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Improve all join perfromance by append `RowRefList` or `RowRef` to AddedColumns for lazy output, while buildOutput, we use `RowRefList`/`RowRef` for output, and remove `is_join_get` condition from `buildOutput` for loop. [#63677](https://github.com/ClickHouse/ClickHouse/pull/63677) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Load filesystem cache metadata asynchronously to boot process, in order to make restarts faster. [#65736](https://github.com/ClickHouse/ClickHouse/pull/65736) ([Daniel Pozo Escalona](https://github.com/danipozo)).
|
||||
* Functions `array` and `map` were optimized to process certain common cases much faster. [#67707](https://github.com/ClickHouse/ClickHouse/pull/67707) ([李扬](https://github.com/taiyang-li)).
|
||||
* Trivial optimize on orc string reading especially when column contains no NULLs. [#67794](https://github.com/ClickHouse/ClickHouse/pull/67794) ([李扬](https://github.com/taiyang-li)).
|
||||
* Improved overall performance of merges by reducing the overhead of scheduling steps of merges. [#68016](https://github.com/ClickHouse/ClickHouse/pull/68016) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Speed up requests to S3 when a profile is not set, credentials are not set, and IMDS is not available (for example, when you are querying a public bucket on a machine outside of a cloud). This closes [#52771](https://github.com/ClickHouse/ClickHouse/issues/52771). [#68082](https://github.com/ClickHouse/ClickHouse/pull/68082) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* New algorithm to determine the unit of marks distribution between replicas by consistent hash. Different numbers of marks chosen for different read patterns to improve performance. [#68424](https://github.com/ClickHouse/ClickHouse/pull/68424) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Try to devirtualize format reader in RowInputFormatWithNamesAndTypes, and see if it could bring some performance improvement. [#68437](https://github.com/ClickHouse/ClickHouse/pull/68437) ([李扬](https://github.com/taiyang-li)).
|
||||
* Add the parallel merge with key implementation to maximize the CPU utilization. [#68441](https://github.com/ClickHouse/ClickHouse/pull/68441) ([Jiebin Sun](https://github.com/jiebinn)).
|
||||
* Add settings `output_format_orc_dictionary_key_size_threshold` to allow user to enable dict encoding for string column in ORC output format. It helps reduce the output orc file size and improve reading performance significantly. [#68591](https://github.com/ClickHouse/ClickHouse/pull/68591) ([李扬](https://github.com/taiyang-li)).
|
||||
* Implemented reading of required files only during hive partitioning. [#68963](https://github.com/ClickHouse/ClickHouse/pull/68963) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Introduce new Keeper request RemoveRecursive which removes node with all it's subtree. [#69332](https://github.com/ClickHouse/ClickHouse/pull/69332) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Speedup insert performance with vector similarity index by adding data to vector index parallel. [#69493](https://github.com/ClickHouse/ClickHouse/pull/69493) ([flynn](https://github.com/ucasfl)).
|
||||
* Previously the algorithmic complexity of part deduplication logic in parallel replica announcement handling was O(n^2) which could take noticeable time for tables with many part (or partitions). This change makes the complexity O(n*log(n)). [#69596](https://github.com/ClickHouse/ClickHouse/pull/69596) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
|
||||
#### Improvement
|
||||
* Hardened parts of the codebase related to parsing of small entities. The following (minor) bugs were found and fixed: - if a `DeltaLake` table is partitioned by Bool, the partition value is always interpreted as false; - `ExternalDistributed` table was using only a single shard in the provided addresses; the value of `max_threads` setting and similar were printed as `'auto(N)'` instead of `auto(N)`. [#52503](https://github.com/ClickHouse/ClickHouse/pull/52503) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Refreshable materialized view improvements: append mode (`... REFRESH EVERY 1 MINUTE APPEND ...`) to add rows to existing table instead of overwriting the whole table, retries (disabled by default, configured in SETTINGS section of the query), `SYSTEM WAIT VIEW <name>` query that waits for the currently running refresh, some fixes. [#58934](https://github.com/ClickHouse/ClickHouse/pull/58934) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Use cgroup-specific metrics for CPU usage accounting instead of system-wide metrics. [#62003](https://github.com/ClickHouse/ClickHouse/pull/62003) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* IO scheduling for remote S3 disks is now done on the level of HTTP socket streams (instead of the whole S3 requests) to resolve `bandwidth_limit` throttling issues. [#65182](https://github.com/ClickHouse/ClickHouse/pull/65182) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Allow a user to have multiple authentication methods instead of only one. Allow authentication methods to be reset to most recently added method. If you want to run instances on 24.8 and one on 24.9 for some time, it's better to set `max_authentication_methods_per_user = 1` for that period to avoid potential errors. [#65277](https://github.com/ClickHouse/ClickHouse/pull/65277) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and characters in arbitrary languages are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#65761](https://github.com/ClickHouse/ClickHouse/pull/65761) ([李扬](https://github.com/taiyang-li)).
|
||||
* When lightweight delete happens on a table with projection(s), despite users have choices either throw an exception (by default) or drop the projection lightweight delete would happen, now the third option is to still have lightweight delete and then rebuild projection(s). [#66169](https://github.com/ClickHouse/ClickHouse/pull/66169) ([Shichao](https://github.com/jsc0218)).
|
||||
* Two options (`dns_allow_resolve_names_to_ipv4` and `dns_allow_resolve_names_to_ipv6`) have been added, to allow block connections ip family. [#66895](https://github.com/ClickHouse/ClickHouse/pull/66895) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Added `min_max`as a new type of (experimental) statistics. It supports estimating range predicates over numeric columns, e.g. `x < 100`. [#67013](https://github.com/ClickHouse/ClickHouse/pull/67013) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Make C-z ignorance configurable (ignore_shell_suspend) in clickhouse-client. [#67134](https://github.com/ClickHouse/ClickHouse/pull/67134) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Improve castOrDefault from Variant/Dynamic columns so it works when inner types are not convertable at all. [#67150](https://github.com/ClickHouse/ClickHouse/pull/67150) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve unicode encoding in JSON output formats. Ensures that valid JSON is generated in the case of certain byte sequences in the result data. [#67938](https://github.com/ClickHouse/ClickHouse/pull/67938) ([mwoenker](https://github.com/mwoenker)).
|
||||
* Added profile events for merges and mutations for better introspection. [#68015](https://github.com/ClickHouse/ClickHouse/pull/68015) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Odbc: get http_max_tries from server configuration. [#68128](https://github.com/ClickHouse/ClickHouse/pull/68128) ([Rodolphe Dugé de Bernonville](https://github.com/RodolpheDuge)).
|
||||
* Add wildcard support for user identification in x509 SubjectAltName extension. [#68236](https://github.com/ClickHouse/ClickHouse/pull/68236) ([Marco Vilas Boas](https://github.com/marco-vb)).
|
||||
* Improve schema inference of date times. Now DateTime64 used only when date time has fractional part, otherwise regular DateTime is used. Inference of Date/DateTime is more strict now, especially when `date_time_input_format='best_effort'` to avoid inferring date times from strings in corner cases. [#68382](https://github.com/ClickHouse/ClickHouse/pull/68382) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Delete old code of named collections from dictionaries and substitute it to the new, which allows to use DDL created named collections in dictionaries. Closes [#60936](https://github.com/ClickHouse/ClickHouse/issues/60936), closes [#36890](https://github.com/ClickHouse/ClickHouse/issues/36890). [#68412](https://github.com/ClickHouse/ClickHouse/pull/68412) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Use HTTP/1.1 instead of HTTP/1.0 (set by default) for external HTTP authentication. [#68456](https://github.com/ClickHouse/ClickHouse/pull/68456) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Functions `upperUTF8` and `lowerUTF8` were previously only able to uppercase / lowercase Cyrillic characters. This limitation is now removed and arbitrary characters are uppercased/lowercased. Example: `SELECT upperUTF8('Süden')` now returns `SÜDEN`. [#68523](https://github.com/ClickHouse/ClickHouse/pull/68523) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Added a new set of metrics for Thread Pool introspection, providing deeper insights into thread pool performance and behavior. [#68674](https://github.com/ClickHouse/ClickHouse/pull/68674) ([filimonov](https://github.com/filimonov)).
|
||||
* Support query parameters in async inserts with format `Values`. [#68741](https://github.com/ClickHouse/ClickHouse/pull/68741) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Support Date32 on dateTrunc and toStartOfInterval. [#68874](https://github.com/ClickHouse/ClickHouse/pull/68874) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Add a new setting: `output_format_always_quote_identifiers` to always quote identifiers if it is enabled. - Add a new setting: `output_format_identifier_quoting_style` to set the identifier quoting style. Applicable values: `'None'`, `'Backticks'` (default), `'DoubleQuotes'`, `'BackticksMySQL'`. [#68896](https://github.com/ClickHouse/ClickHouse/pull/68896) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Add `plan_step_name` and `plan_step_description` columns to `system.processors_profile_log`. [#68954](https://github.com/ClickHouse/ClickHouse/pull/68954) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Support for the Spanish language in the embedded dictionaries. [#69035](https://github.com/ClickHouse/ClickHouse/pull/69035) ([Vasily Okunev](https://github.com/VOkunev)).
|
||||
* Add CPU arch to short fault info. [#69037](https://github.com/ClickHouse/ClickHouse/pull/69037) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses. Now it supports only for MergeTree family of table engines. [#69076](https://github.com/ClickHouse/ClickHouse/pull/69076) ([sakulali](https://github.com/sakulali)).
|
||||
* Replication of subset of columns is now available through MaterializedPostgreSQL. Closes [#33748](https://github.com/ClickHouse/ClickHouse/issues/33748). [#69092](https://github.com/ClickHouse/ClickHouse/pull/69092) ([Kruglov Kirill](https://github.com/1on)).
|
||||
* Add `std::string getHost(const std::string & key) const;` method to "base/poco/Util/include/Poco/Util/AbstractConfiguration.h" which does the same as `Poco::Util::AbstractConfiguration::getString` method but additionally does validity check to make sure value is a correct IP address or domain name. [#69130](https://github.com/ClickHouse/ClickHouse/pull/69130) ([Maxim](https://github.com/m4xxx1m)).
|
||||
* Do not block retries when establishing a new keeper connection. [#69148](https://github.com/ClickHouse/ClickHouse/pull/69148) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Update DatabaseFactory so it would be possible for user defined database engines to have arguments, settings and table overrides (similar to StorageFactory). [#69201](https://github.com/ClickHouse/ClickHouse/pull/69201) ([Барыкин Никита Олегович](https://github.com/NikBarykin)).
|
||||
* Restore mode that replaces all external table engines and functions to Null (`restore_replace_external_engines_to_null`, `restore_replace_external_table_functions_to_null` settings) was failing if table had SETTINGS. Now it removes settings from table definition in this case and allows to restore such tables. [#69253](https://github.com/ClickHouse/ClickHouse/pull/69253) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Reduce memory usage of inserts to JSON by using adaptive write buffer size. A lot of files created by JSON column in wide part contains small amount of data and it doesn't make sense to allocate 1MB buffer for them. [#69272](https://github.com/ClickHouse/ClickHouse/pull/69272) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Added user-level settings `min_free_disk_bytes_to_throw_insert` and `min_free_disk_ratio_to_throw_insert` to prevent insertions on disks that are almost full. [#69376](https://github.com/ClickHouse/ClickHouse/pull/69376) ([Marco Vilas Boas](https://github.com/marco-vb)).
|
||||
* Not retaining thread in concurrent hash join threadpool to avoid query excessively spawn threads. [#69406](https://github.com/ClickHouse/ClickHouse/pull/69406) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Allow empty arguments for arrayZip/arrayZipUnaligned, as concat did in https://github.com/ClickHouse/ClickHouse/pull/65887. It is for spark compatiability in Gluten CH Backend. [#69576](https://github.com/ClickHouse/ClickHouse/pull/69576) ([李扬](https://github.com/taiyang-li)).
|
||||
* Support more advanced SSL options for Keeper's internal communication (e.g. private keys with passphrase). [#69582](https://github.com/ClickHouse/ClickHouse/pull/69582) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Sleep for 10ms before retrying to acquire a lock in `databasereplicatedddlworker::enqueuequeryimpl`. [#69588](https://github.com/ClickHouse/ClickHouse/pull/69588) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Index analysis can take noticeable time for big tables with many parts or partitions. This change should enable killing a heavy query at that stage. [#69606](https://github.com/ClickHouse/ClickHouse/pull/69606) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Masking sensitive info in `gcs()` table function. [#69611](https://github.com/ClickHouse/ClickHouse/pull/69611) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Sync changes to `interpreterdropquery::executetodatabaseimpl` from the private fork. [#69670](https://github.com/ClickHouse/ClickHouse/pull/69670) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69905](https://github.com/ClickHouse/ClickHouse/issues/69905): Change the join to sort settings type to unsigned int. [#69886](https://github.com/ClickHouse/ClickHouse/pull/69886) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Rebuild projection for merges that reduce number of rows. [#62364](https://github.com/ClickHouse/ClickHouse/pull/62364) ([cangyin](https://github.com/cangyin)).
|
||||
* Fix attaching table when pg dbname contains "-" in MaterializedPostgreSQL. [#62730](https://github.com/ClickHouse/ClickHouse/pull/62730) ([takakawa](https://github.com/takakawa)).
|
||||
* Storage Join support for nullable columns in left table, close [#61247](https://github.com/ClickHouse/ClickHouse/issues/61247). [#66926](https://github.com/ClickHouse/ClickHouse/pull/66926) ([vdimir](https://github.com/vdimir)).
|
||||
* Incorrect query result with parallel replicas (distribute queries as well) when `IN` operator contains conversion to Decimal(). The bug was introduced with the new analyzer. [#67234](https://github.com/ClickHouse/ClickHouse/pull/67234) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix the problem that alter modfiy order by causes inconsistent metadata. [#67436](https://github.com/ClickHouse/ClickHouse/pull/67436) ([iceFireser](https://github.com/iceFireser)).
|
||||
* Fix the upper bound of the function `fromModifiedJulianDay`. It was supposed to be `9999-12-31` but was mistakenly set to `9999-01-01`. [#67583](https://github.com/ClickHouse/ClickHouse/pull/67583) ([PHO](https://github.com/depressed-pho)).
|
||||
* Fix when the index is not at the beginning of the tuple during `IN` query. [#67626](https://github.com/ClickHouse/ClickHouse/pull/67626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fixed error on generated columns in MaterializedPostgreSQL when adnum ordering is broken [#63161](https://github.com/ClickHouse/ClickHouse/issues/63161). Fixed error on id column with nextval expression as default MaterializedPostgreSQL when there are generated columns in table. Fixed error on dropping publication with symbols except [a-z1-9-]. [#67664](https://github.com/ClickHouse/ClickHouse/pull/67664) ([Kruglov Kirill](https://github.com/1on)).
|
||||
* Fix expiration in `RoleCache`. [#67748](https://github.com/ClickHouse/ClickHouse/pull/67748) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix window view missing blocks due to slow flush to view. [#67983](https://github.com/ClickHouse/ClickHouse/pull/67983) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix MSAN issue caused by incorrect date format. [#68105](https://github.com/ClickHouse/ClickHouse/pull/68105) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Fixed crash in Parquet filtering when data types in the file substantially differ from requested types (e.g. `... FROM file('a.parquet', Parquet, 'x String')`, but the file has `x Int64`). Without this fix, use `input_format_parquet_filter_push_down = 0` as a workaround. [#68131](https://github.com/ClickHouse/ClickHouse/pull/68131) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix crash in `lag`/`lead` which is introduced in [#67091](https://github.com/ClickHouse/ClickHouse/issues/67091). [#68262](https://github.com/ClickHouse/ClickHouse/pull/68262) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Try fix postgres crash when query is cancelled. [#68288](https://github.com/ClickHouse/ClickHouse/pull/68288) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* After https://github.com/ClickHouse/ClickHouse/pull/61984 `schema_inference_make_columns_nullable=0` still can make columns `Nullable` in Parquet/Arrow formats. The change was backward incompatible and users noticed the changes in the behaviour. This PR makes `schema_inference_make_columns_nullable=0` to work as before (no Nullable columns will be inferred) and introduces new value `auto` for this setting that will make columns `Nullable` only if data has information about nullability. [#68298](https://github.com/ClickHouse/ClickHouse/pull/68298) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fixes [#50868](https://github.com/ClickHouse/ClickHouse/issues/50868). Small DateTime64 constant values returned by a nested subquery inside a distributed query were wrongly transformed to Nulls, thus causing errors and possible incorrect query results. [#68323](https://github.com/ClickHouse/ClickHouse/pull/68323) ([Shankar](https://github.com/shiyer7474)).
|
||||
* Fix missing sync replica mode in query `SYSTEM SYNC REPLICA`. [#68326](https://github.com/ClickHouse/ClickHouse/pull/68326) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix bug in key condition. [#68354](https://github.com/ClickHouse/ClickHouse/pull/68354) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix crash on drop or rename a role that is used in LDAP external user directory. [#68355](https://github.com/ClickHouse/ClickHouse/pull/68355) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Fix Progress column value of system.view_refreshes greater than 1 [#68377](https://github.com/ClickHouse/ClickHouse/issues/68377). [#68378](https://github.com/ClickHouse/ClickHouse/pull/68378) ([megao](https://github.com/jetgm)).
|
||||
* Process regexp flags correctly. [#68389](https://github.com/ClickHouse/ClickHouse/pull/68389) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* PostgreSQL-style cast operator (`::`) works correctly even for SQL-style hex and binary string literals (e.g., `SELECT x'414243'::String`). This closes [#68324](https://github.com/ClickHouse/ClickHouse/issues/68324). [#68482](https://github.com/ClickHouse/ClickHouse/pull/68482) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Minor patch for https://github.com/ClickHouse/ClickHouse/pull/68131. [#68494](https://github.com/ClickHouse/ClickHouse/pull/68494) ([Chang chen](https://github.com/baibaichen)).
|
||||
* Fix [#68239](https://github.com/ClickHouse/ClickHouse/issues/68239) SAMPLE n where n is an integer. [#68499](https://github.com/ClickHouse/ClickHouse/pull/68499) ([Denis Hananein](https://github.com/denis-hananein)).
|
||||
* Fix bug in mann-whitney-utest when the size of two districutions are not equal. [#68556](https://github.com/ClickHouse/ClickHouse/pull/68556) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* After unexpected restart, fail to start replication of ReplicatedMergeTree due to abnormal handling of covered-by-broken part. [#68584](https://github.com/ClickHouse/ClickHouse/pull/68584) ([baolin](https://github.com/baolinhuang)).
|
||||
* Fix `LOGICAL_ERROR`s when functions `sipHash64Keyed`, `sipHash128Keyed`, or `sipHash128ReferenceKeyed` are applied to empty arrays or tuples. [#68630](https://github.com/ClickHouse/ClickHouse/pull/68630) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Full text index may filter out wrong columns when index multiple columns, it didn't reset row_id between different columns, the reproduce procedure is in tests/queries/0_stateless/03228_full_text_with_multi_col.sql. Without this. [#68644](https://github.com/ClickHouse/ClickHouse/pull/68644) ([siyuan](https://github.com/linkwk7)).
|
||||
* Fix invalid character '\t' and '\n' in replica_name when creating a Replicated table, which causes incorrect parsing of 'source replica' in LogEntry. Mentioned in issue [#68640](https://github.com/ClickHouse/ClickHouse/issues/68640). [#68645](https://github.com/ClickHouse/ClickHouse/pull/68645) ([Zhigao Hong](https://github.com/zghong)).
|
||||
* Added back virtual columns ` _table` and `_database` to distributed tables. They were available until version 24.3. [#68672](https://github.com/ClickHouse/ClickHouse/pull/68672) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix possible error `Size of permutation (0) is less than required (...)` during Variant column permutation. [#68681](https://github.com/ClickHouse/ClickHouse/pull/68681) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix possible error `DB::Exception: Block structure mismatch in joined block stream: different columns:` with new JSON column. [#68686](https://github.com/ClickHouse/ClickHouse/pull/68686) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix issue with materialized constant keys when hashing maps with arrays as keys in functions `sipHash(64/128)Keyed`. [#68731](https://github.com/ClickHouse/ClickHouse/pull/68731) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Make `ColumnsDescription::toString` format each column using the same `IAST::FormatState object`. This results in uniform columns metadata being written to disk and ZooKeeper. [#68733](https://github.com/ClickHouse/ClickHouse/pull/68733) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Fix merging of aggregated data for grouping sets. [#68744](https://github.com/ClickHouse/ClickHouse/pull/68744) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix logical error, when we create a replicated merge tree, alter a column and then execute modify statistics. [#68820](https://github.com/ClickHouse/ClickHouse/pull/68820) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix resolving dynamic subcolumns from subqueries in analyzer. [#68824](https://github.com/ClickHouse/ClickHouse/pull/68824) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix complex types metadata parsing in DeltaLake. Closes [#68739](https://github.com/ClickHouse/ClickHouse/issues/68739). [#68836](https://github.com/ClickHouse/ClickHouse/pull/68836) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed asynchronous inserts in case when metadata of table is changed (by `ALTER ADD/MODIFY COLUMN` queries) after insert but before flush to the table. [#68837](https://github.com/ClickHouse/ClickHouse/pull/68837) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix unexpected exception when passing empty tuple in array. This fixes [#68618](https://github.com/ClickHouse/ClickHouse/issues/68618). [#68848](https://github.com/ClickHouse/ClickHouse/pull/68848) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix parsing pure metadata mutations commands. [#68935](https://github.com/ClickHouse/ClickHouse/pull/68935) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Fix possible wrong result during anyHeavy state merge. [#68950](https://github.com/ClickHouse/ClickHouse/pull/68950) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fixed writing to Materialized Views with enabled setting `optimize_functions_to_subcolumns`. [#68951](https://github.com/ClickHouse/ClickHouse/pull/68951) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Don't use serializations cache in const Dynamic column methods. It could let to use-of-unitialized value or even race condition during aggregations. [#68953](https://github.com/ClickHouse/ClickHouse/pull/68953) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix parsing error when null should be inserted as default in some cases during JSON type parsing. [#68955](https://github.com/ClickHouse/ClickHouse/pull/68955) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* There were cases when path was concatenated incorrectly and had the `//` part in it, solving this problem using path normalization. [#69066](https://github.com/ClickHouse/ClickHouse/pull/69066) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix logical error when we have empty async insert. [#69080](https://github.com/ClickHouse/ClickHouse/pull/69080) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fixed data race of progress indication in clickhouse-client during query canceling. [#69081](https://github.com/ClickHouse/ClickHouse/pull/69081) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Fix a bug that the vector similarity index (currently experimental) was not utilized when used with cosine distance as distance function. [#69090](https://github.com/ClickHouse/ClickHouse/pull/69090) ([flynn](https://github.com/ucasfl)).
|
||||
* This change addresses an issue where attempting to create a Replicated database again after a server failure during the initial creation process could result in error. [#69102](https://github.com/ClickHouse/ClickHouse/pull/69102) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Don't infer Bool type from String in CSV when `input_format_csv_try_infer_numbers_from_strings = 1` because we don't allow reading bool values from strings. [#69109](https://github.com/ClickHouse/ClickHouse/pull/69109) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix explain ast insert queries parsing errors on client when `--multiquery` is enabled. [#69123](https://github.com/ClickHouse/ClickHouse/pull/69123) ([wxybear](https://github.com/wxybear)).
|
||||
* `UNION` clause in subqueries wasn't handled correctly in queries with parallel replicas and lead to LOGICAL_ERROR `Duplicate announcement received for replica`. [#69146](https://github.com/ClickHouse/ClickHouse/pull/69146) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix propogating structure argument in s3Cluster. Previously the `DEFAULT` expression of the column could be lost when sending the query to the replicas in s3Cluster. [#69147](https://github.com/ClickHouse/ClickHouse/pull/69147) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Respect format settings in Values format during conversion from expression to the destination type. [#69149](https://github.com/ClickHouse/ClickHouse/pull/69149) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix `clickhouse-client --queries-file` for readonly users (previously fails with `Cannot modify 'log_comment' setting in readonly mode`). [#69175](https://github.com/ClickHouse/ClickHouse/pull/69175) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix data race in clickhouse-client when it's piped to a process that terminated early. [#69186](https://github.com/ClickHouse/ClickHouse/pull/69186) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix incorrect results of Fix uniq and GROUP BY for JSON/Dynamic types. [#69203](https://github.com/ClickHouse/ClickHouse/pull/69203) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix the INFILE format detection for asynchronous inserts. If the format is not explicitly defined in the FORMAT clause, it can be detected from the INFILE file extension. [#69237](https://github.com/ClickHouse/ClickHouse/pull/69237) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* After [this issue](https://github.com/ClickHouse/ClickHouse/pull/59946#issuecomment-1943653197) there are quite a few table replicas in production such that their `metadata_version` node value is both equal to `0` and is different from the respective table's `metadata` node version. This leads to `alter` queries failing on such replicas. [#69274](https://github.com/ClickHouse/ClickHouse/pull/69274) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#69986](https://github.com/ClickHouse/ClickHouse/issues/69986): Fix inf loop after `restore replica` in the replicated merge tree with zero copy. [#69293](https://github.com/ClickHouse/ClickHouse/pull/69293) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Mark Dynamic type as not safe primary key type to avoid issues with Fields. [#69311](https://github.com/ClickHouse/ClickHouse/pull/69311) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve restoring of access entities' dependencies. [#69346](https://github.com/ClickHouse/ClickHouse/pull/69346) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Fix undefined behavior when all connection attempts fail getting a connection for insertions. [#69390](https://github.com/ClickHouse/ClickHouse/pull/69390) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Close [#69135](https://github.com/ClickHouse/ClickHouse/issues/69135). If we try to reuse joined data for `cross` join, but this could not happen in ClickHouse at present. It's better to keep `have_compressed` in `reuseJoinedData`. [#69404](https://github.com/ClickHouse/ClickHouse/pull/69404) ([lgbo](https://github.com/lgbo-ustc)).
|
||||
* Make `materialize()` function return full column when parameter is a sparse column. [#69429](https://github.com/ClickHouse/ClickHouse/pull/69429) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fixed a `LOGICAL_ERROR` with function `sqidDecode` ([#69450](https://github.com/ClickHouse/ClickHouse/issues/69450)). [#69451](https://github.com/ClickHouse/ClickHouse/pull/69451) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Quick fix for s3queue problem on 24.6 or create query with database replicated. [#69454](https://github.com/ClickHouse/ClickHouse/pull/69454) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed case when memory consumption was too high because of the squashing in `INSERT INTO ... SELECT` or `CREATE TABLE AS SELECT` queries. [#69469](https://github.com/ClickHouse/ClickHouse/pull/69469) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Statements `SHOW COLUMNS` and `SHOW INDEX` now work properly if the table has dots in its name. [#69514](https://github.com/ClickHouse/ClickHouse/pull/69514) ([Salvatore Mesoraca](https://github.com/aiven-sal)).
|
||||
* Usage of the query cache for queries with an overflow mode != 'throw' is now disallowed. This prevents situations where potentially truncated and incorrect query results could be stored in the query cache. (issue [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476)). [#69549](https://github.com/ClickHouse/ClickHouse/pull/69549) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Keep original order of conditions during move to prewhere. Previously the order could change and it could lead to failing queries when the order is important. [#69560](https://github.com/ClickHouse/ClickHouse/pull/69560) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix Keeper multi-request preprocessing after ZNOAUTH error. [#69627](https://github.com/ClickHouse/ClickHouse/pull/69627) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix METADATA_MISMATCH that might have happened due to TTL with a WHERE clause in DatabaseReplicated when creating a new replica. [#69736](https://github.com/ClickHouse/ClickHouse/pull/69736) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix `StorageS3(Azure)Queue` settings `tracked_file_ttl_sec`. We wrote it to keeper with key `tracked_file_ttl_sec`, but read as `tracked_files_ttl_sec`, which was a typo. [#69742](https://github.com/ClickHouse/ClickHouse/pull/69742) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Make getHyperrectangleForRowGroup not throw an exception when the data type in parquet file is not convertable into the requested data type. Solved the user's problem when the Parquet file had Decimal64 data type and the column data type was DateTime. [#69745](https://github.com/ClickHouse/ClickHouse/pull/69745) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* Backported in [#70021](https://github.com/ClickHouse/ClickHouse/issues/70021): Fix analyzer default with old compatibility value. [#69895](https://github.com/ClickHouse/ClickHouse/pull/69895) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69943](https://github.com/ClickHouse/ClickHouse/issues/69943): Don't check dependencies during CREATE OR REPLACE VIEW during DROP of old table. Previously CREATE OR REPLACE query failed when there are dependent tables of the recreated view. [#69907](https://github.com/ClickHouse/ClickHouse/pull/69907) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70003](https://github.com/ClickHouse/ClickHouse/issues/70003): Now SQL security will work with parameterized views correctly. [#69984](https://github.com/ClickHouse/ClickHouse/pull/69984) ([pufit](https://github.com/pufit)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Allow to specify min and max for random settings in the test after tags in a form `setting=(min, max)`. For some tests really low value of a setting can lead to timeout (for example `index_granularity=1`) but we still want to randomize this setting in such tests. To avoid timeouts we can specify custom minimum value for this setting. [#67875](https://github.com/ClickHouse/ClickHouse/pull/67875) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Integration tests flaky check now runs each module(file) as a separate group. It allows to apply timeout per module. Previously slow tests could use all the time and other tests were not run in flaky check with successful result. [#68380](https://github.com/ClickHouse/ClickHouse/pull/68380) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Backported in [#69980](https://github.com/ClickHouse/ClickHouse/issues/69980): Makes dbms independent from clickhouse_functions. [#69914](https://github.com/ClickHouse/ClickHouse/pull/69914) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NO CL CATEGORY
|
||||
|
||||
* Backported in [#69995](https://github.com/ClickHouse/ClickHouse/issues/69995):. [#69989](https://github.com/ClickHouse/ClickHouse/pull/69989) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### NO CL ENTRY
|
||||
|
||||
* NO CL ENTRY: 'Revert "Revert "Fix AWS ECS""'. [#65362](https://github.com/ClickHouse/ClickHouse/pull/65362) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* NO CL ENTRY: 'Revert "[RFC] Fix settings/current_database in system.processes for async BACKUP/RESTORE"'. [#68386](https://github.com/ClickHouse/ClickHouse/pull/68386) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* NO CL ENTRY: 'Revert "Improve compatibility of `upper/lowerUTF8` with Spark"'. [#68510](https://github.com/ClickHouse/ClickHouse/pull/68510) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* NO CL ENTRY: 'Revert "Fix unexpected behavior with `FORMAT` and `SETTINGS` parsing"'. [#68608](https://github.com/ClickHouse/ClickHouse/pull/68608) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Revert "Fix prewhere without columns and without adaptive index granularity (almost w/o anything)"'. [#68897](https://github.com/ClickHouse/ClickHouse/pull/68897) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* NO CL ENTRY: 'Revert "Speed up some Kafka tests with multiprocessing"'. [#69356](https://github.com/ClickHouse/ClickHouse/pull/69356) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* NO CL ENTRY: 'Revert "Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V"'. [#69393](https://github.com/ClickHouse/ClickHouse/pull/69393) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Revert "Add user-level settings min_free_diskspace_bytes_to_throw_insert and min_free_diskspace_ratio_to_throw_insert"'. [#69705](https://github.com/ClickHouse/ClickHouse/pull/69705) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* NO CL ENTRY: 'Revert "Support more oss endpoints"'. [#69779](https://github.com/ClickHouse/ClickHouse/pull/69779) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Followup for [#56996](https://github.com/ClickHouse/ClickHouse/issues/56996). [#58289](https://github.com/ClickHouse/ClickHouse/pull/58289) ([vdimir](https://github.com/vdimir)).
|
||||
* Add chunked wrapper to native protocol. [#63781](https://github.com/ClickHouse/ClickHouse/pull/63781) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Local plan for parallel replicas. [#64448](https://github.com/ClickHouse/ClickHouse/pull/64448) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Disallow creating refreshable MV on Linux < 3.15. [#64953](https://github.com/ClickHouse/ClickHouse/pull/64953) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Allow query profiler period to be longer than 4 seconds. [#65255](https://github.com/ClickHouse/ClickHouse/pull/65255) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Improved getting of alter conversions for queries. [#65832](https://github.com/ClickHouse/ClickHouse/pull/65832) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Simplify some memory tracking parts. [#66648](https://github.com/ClickHouse/ClickHouse/pull/66648) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* What if we tighten limits for functional tests?. [#66837](https://github.com/ClickHouse/ClickHouse/pull/66837) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix flaky `test_delayed_replica_failover`. [#67541](https://github.com/ClickHouse/ClickHouse/pull/67541) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Fix(asan): access destroyed shared context from handleCrash(). [#67700](https://github.com/ClickHouse/ClickHouse/pull/67700) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. II. [#67749](https://github.com/ClickHouse/ClickHouse/pull/67749) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Make `ColumnLowCardinality::getName()` consistent with other columns. [#67789](https://github.com/ClickHouse/ClickHouse/pull/67789) ([李扬](https://github.com/taiyang-li)).
|
||||
* Catch exception in destructor of LocalFileHolder. [#67891](https://github.com/ClickHouse/ClickHouse/pull/67891) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Add keeper error description to the message. [#67935](https://github.com/ClickHouse/ClickHouse/pull/67935) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Timeout handling for functional and integration tests, store artifacts and report if timed out - sets 2h default timeout for all jobs. [#67944](https://github.com/ClickHouse/ClickHouse/pull/67944) ([Max K.](https://github.com/maxknv)).
|
||||
* Add test cases to 03217_datetime64_constant_to_ast. [#67966](https://github.com/ClickHouse/ClickHouse/pull/67966) ([vdimir](https://github.com/vdimir)).
|
||||
* Remove some no-parallel tags from tests (Part 6). [#68048](https://github.com/ClickHouse/ClickHouse/pull/68048) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Keeper improvements package. [#68108](https://github.com/ClickHouse/ClickHouse/pull/68108) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* CI: Docker images clean from test scripts. [#68117](https://github.com/ClickHouse/ClickHouse/pull/68117) ([Max K.](https://github.com/maxknv)).
|
||||
* Output an operation error for ZK Multi request failed operation into log. [#68127](https://github.com/ClickHouse/ClickHouse/pull/68127) ([Aleksei Filatov](https://github.com/aalexfvk)).
|
||||
* Better test for Not-ready Set is passed in system.* tables. [#68139](https://github.com/ClickHouse/ClickHouse/pull/68139) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* ci: add more logs in the functional tests reports. [#68153](https://github.com/ClickHouse/ClickHouse/pull/68153) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix fundamentally broken `test_throttling`. [#68180](https://github.com/ClickHouse/ClickHouse/pull/68180) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `00600_replace_running_query`. [#68184](https://github.com/ClickHouse/ClickHouse/pull/68184) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix non-deterministic result order in `test_storage_mysql.test_mysql_distributed`. [#68188](https://github.com/ClickHouse/ClickHouse/pull/68188) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try to unflaky `test_storage_s3_queue/test.py::test_shards_distributed`. [#68233](https://github.com/ClickHouse/ClickHouse/pull/68233) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Bump usearch to v2.13.2. [#68248](https://github.com/ClickHouse/ClickHouse/pull/68248) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix 'Refresh set entry already exists'. [#68249](https://github.com/ClickHouse/ClickHouse/pull/68249) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* tests: make 01600_parts_states_metrics_long better. [#68265](https://github.com/ClickHouse/ClickHouse/pull/68265) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Avoid ignoring errors of execute_process() (set COMMAND_ERROR_IS_FATAL=ANY). [#68267](https://github.com/ClickHouse/ClickHouse/pull/68267) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix FullSortingJoinTest.AsofGreaterGeneratedTestData with empty data. [#68280](https://github.com/ClickHouse/ClickHouse/pull/68280) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix min/max time columns in TimeSeries table. [#68282](https://github.com/ClickHouse/ClickHouse/pull/68282) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Update fuzzer dictionary. [#68284](https://github.com/ClickHouse/ClickHouse/pull/68284) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* [Green CI] Test 00652_mergetree_mutations is flaky. [#68286](https://github.com/ClickHouse/ClickHouse/pull/68286) ([Daniil Ivanik](https://github.com/divanik)).
|
||||
* Fix test storage_join_direct_join. [#68287](https://github.com/ClickHouse/ClickHouse/pull/68287) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Add execution status to PipelineExecutor. Avoid LOGICAL_ERROR if the pushing pipeline was canceled by timeout. Replace `Pipeline for PushingPipelineExecutor was finished before all data was inserted. (LOGICAL_ERROR)` to `QUERY_WAS_CANCELLED`. Closes [#63979](https://github.com/ClickHouse/ClickHouse/issues/63979). [#68291](https://github.com/ClickHouse/ClickHouse/pull/68291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* CI: Create new release branch workflow updates. [#68294](https://github.com/ClickHouse/ClickHouse/pull/68294) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version after release. [#68306](https://github.com/ClickHouse/ClickHouse/pull/68306) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Various vector similarity index related fixes. [#68308](https://github.com/ClickHouse/ClickHouse/pull/68308) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix: min marks to read overflow with parallel replicas. [#68309](https://github.com/ClickHouse/ClickHouse/pull/68309) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Apply libunwind changes needed for musl. [#68312](https://github.com/ClickHouse/ClickHouse/pull/68312) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update musl to 1.2.5 with unwind info. [#68313](https://github.com/ClickHouse/ClickHouse/pull/68313) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix data race on SampleKey. [#68321](https://github.com/ClickHouse/ClickHouse/pull/68321) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix part name in 00961_check_table. [#68325](https://github.com/ClickHouse/ClickHouse/pull/68325) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix 02995_index_10 timeout. [#68329](https://github.com/ClickHouse/ClickHouse/pull/68329) ([vdimir](https://github.com/vdimir)).
|
||||
* CI: Fix for change log critical bug fix regex. [#68332](https://github.com/ClickHouse/ClickHouse/pull/68332) ([Max K.](https://github.com/maxknv)).
|
||||
* Optionally re-enable compilation with `-O0`. [#68352](https://github.com/ClickHouse/ClickHouse/pull/68352) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add debug info for `00180_no_seek_avoiding_when_reading_from_cache`. [#68353](https://github.com/ClickHouse/ClickHouse/pull/68353) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix data race in `DynamicResourceManager::updateConfiguration`. [#68356](https://github.com/ClickHouse/ClickHouse/pull/68356) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Update version_date.tsv and changelog after v24.3.7.30-lts. [#68357](https://github.com/ClickHouse/ClickHouse/pull/68357) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* More improvements in integration tests. [#68358](https://github.com/ClickHouse/ClickHouse/pull/68358) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Rename: S3DiskNoKeyErrors -> DiskS3NoSuchKeyErrors. [#68361](https://github.com/ClickHouse/ClickHouse/pull/68361) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
* CI: Minor fixes for changelog and release exceptions. [#68362](https://github.com/ClickHouse/ClickHouse/pull/68362) ([Max K.](https://github.com/maxknv)).
|
||||
* Check that setProcessListElement() is not called on global context. [#68364](https://github.com/ClickHouse/ClickHouse/pull/68364) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix off-by-one inline function info in stack traces. [#68365](https://github.com/ClickHouse/ClickHouse/pull/68365) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Check that merge entries are valid. [#68366](https://github.com/ClickHouse/ClickHouse/pull/68366) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* performance comparison test for output_format_parquet_write_page_index. [#68367](https://github.com/ClickHouse/ClickHouse/pull/68367) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Add a test for [#57324](https://github.com/ClickHouse/ClickHouse/issues/57324). [#68376](https://github.com/ClickHouse/ClickHouse/pull/68376) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Auto release workflow. [#68402](https://github.com/ClickHouse/ClickHouse/pull/68402) ([Max K.](https://github.com/maxknv)).
|
||||
* Update delta lake test. [#68404](https://github.com/ClickHouse/ClickHouse/pull/68404) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add a test for [#59118](https://github.com/ClickHouse/ClickHouse/issues/59118). [#68406](https://github.com/ClickHouse/ClickHouse/pull/68406) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. IV. [#68407](https://github.com/ClickHouse/ClickHouse/pull/68407) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try to fix test 03221_mutation_analyzer_skip_part. [#68409](https://github.com/ClickHouse/ClickHouse/pull/68409) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Add `NotStarted` status to not-prepared processors. This is for better diagnostics in the case of PipelineStuck. [#68413](https://github.com/ClickHouse/ClickHouse/pull/68413) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Update log message. [#68426](https://github.com/ClickHouse/ClickHouse/pull/68426) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix 01119_session_log flakiness. [#68433](https://github.com/ClickHouse/ClickHouse/pull/68433) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Try to mitigate 02818_memory_profiler_sample_min_max_allocation_size flakiness. [#68434](https://github.com/ClickHouse/ClickHouse/pull/68434) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update analyzer_tech_debt.txt. [#68443](https://github.com/ClickHouse/ClickHouse/pull/68443) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove obsolete `-n` / `--multiquery` from tests. [#68447](https://github.com/ClickHouse/ClickHouse/pull/68447) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Check for invalid regexp in JSON SKIP REGEXP section. [#68451](https://github.com/ClickHouse/ClickHouse/pull/68451) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Better inference of date times 2. [#68452](https://github.com/ClickHouse/ClickHouse/pull/68452) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CI: Native build for package_aarch64. [#68457](https://github.com/ClickHouse/ClickHouse/pull/68457) ([Max K.](https://github.com/maxknv)).
|
||||
* Minor update in Dynamic/JSON serializations. [#68459](https://github.com/ClickHouse/ClickHouse/pull/68459) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix test `02122_join_group_by_timeout`. [#68462](https://github.com/ClickHouse/ClickHouse/pull/68462) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add check for `min_number_of_marks = 0` in parallel replicas requests. [#68476](https://github.com/ClickHouse/ClickHouse/pull/68476) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix `Broken pipe` error for `03149_numbers_max_block_size_zero.sh`. [#68478](https://github.com/ClickHouse/ClickHouse/pull/68478) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Use temporary tables for input and output in `clickhouse-local`. [#68483](https://github.com/ClickHouse/ClickHouse/pull/68483) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Pass-through RENAME and UUID-related operations in Overlay database to underlying databases. [#68486](https://github.com/ClickHouse/ClickHouse/pull/68486) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix output of clickhouse-test in case of tests timeouts. [#68487](https://github.com/ClickHouse/ClickHouse/pull/68487) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Part of [#68024](https://github.com/ClickHouse/ClickHouse/issues/68024). [#68488](https://github.com/ClickHouse/ClickHouse/pull/68488) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous changes in BaseDaemon. [#68489](https://github.com/ClickHouse/ClickHouse/pull/68489) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999) (2). [#68490](https://github.com/ClickHouse/ClickHouse/pull/68490) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellaneous. [#68504](https://github.com/ClickHouse/ClickHouse/pull/68504) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Miscellanous changes from [#66999](https://github.com/ClickHouse/ClickHouse/issues/66999). [#68507](https://github.com/ClickHouse/ClickHouse/pull/68507) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `01017_uniqCombined_memory_usage`. [#68508](https://github.com/ClickHouse/ClickHouse/pull/68508) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix race condition in MergeTreeRestartingThread. [#68513](https://github.com/ClickHouse/ClickHouse/pull/68513) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test `01079_bad_alters_zookeeper_long`. [#68515](https://github.com/ClickHouse/ClickHouse/pull/68515) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix bad exception messages. [#68520](https://github.com/ClickHouse/ClickHouse/pull/68520) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* CI: Auto Releases in prod. [#68525](https://github.com/ClickHouse/ClickHouse/pull/68525) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix build with `-DENABLE_LIBRARIES=0`. [#68527](https://github.com/ClickHouse/ClickHouse/pull/68527) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Removed the use of the context during function execution, as it was causing errors when the context was expired. [#68534](https://github.com/ClickHouse/ClickHouse/pull/68534) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* CI: Minor release workflow fix. [#68536](https://github.com/ClickHouse/ClickHouse/pull/68536) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix after [#68291](https://github.com/ClickHouse/ClickHouse/issues/68291). [#68548](https://github.com/ClickHouse/ClickHouse/pull/68548) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Stateless tests: increase hung check timeout. [#68552](https://github.com/ClickHouse/ClickHouse/pull/68552) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* CI: Tidy build timeout from 2h to 3h. [#68567](https://github.com/ClickHouse/ClickHouse/pull/68567) ([Max K.](https://github.com/maxknv)).
|
||||
* Reduce memory consumption in ghdata JSON tests. [#68571](https://github.com/ClickHouse/ClickHouse/pull/68571) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Move enabling experimental settings to a separate file. [#68577](https://github.com/ClickHouse/ClickHouse/pull/68577) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Minor logging fixes. [#68578](https://github.com/ClickHouse/ClickHouse/pull/68578) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix enumerating dynamic subcolumns. [#68582](https://github.com/ClickHouse/ClickHouse/pull/68582) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Update version_date.tsv and changelog after v23.8.16.16-lts. [#68593](https://github.com/ClickHouse/ClickHouse/pull/68593) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update 02995_index_7.sh. [#68594](https://github.com/ClickHouse/ClickHouse/pull/68594) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix style in Functions/printf.cpp. [#68595](https://github.com/ClickHouse/ClickHouse/pull/68595) ([vdimir](https://github.com/vdimir)).
|
||||
* Update version_date.tsv and changelog after v24.3.8.13-lts. [#68599](https://github.com/ClickHouse/ClickHouse/pull/68599) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Prioritizing of virtual columns in hive partitioning. [#68606](https://github.com/ClickHouse/ClickHouse/pull/68606) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* materialized_view_deduplication performance comparison test. [#68607](https://github.com/ClickHouse/ClickHouse/pull/68607) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Update README.md. [#68610](https://github.com/ClickHouse/ClickHouse/pull/68610) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* patch: fix reference to sorting key in primary key docs. [#68612](https://github.com/ClickHouse/ClickHouse/pull/68612) ([Leon Kozlowski](https://github.com/leonkozlowski)).
|
||||
* Do not fuzz 02835 drop user during session. [#68620](https://github.com/ClickHouse/ClickHouse/pull/68620) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix 03221_s3_imds_decent_timeout. [#68627](https://github.com/ClickHouse/ClickHouse/pull/68627) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix test `01079_bad_alters_zookeeper_long`. [#68629](https://github.com/ClickHouse/ClickHouse/pull/68629) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Disable min_bytes_to_use_direct_io in some tests with Dynamic/JSON subcolumns because it's broken. [#68632](https://github.com/ClickHouse/ClickHouse/pull/68632) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix false "Killed by signal (output files)" in stress_tests.lib. [#68638](https://github.com/ClickHouse/ClickHouse/pull/68638) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Remove wrong release version. [#68646](https://github.com/ClickHouse/ClickHouse/pull/68646) ([Max K.](https://github.com/maxknv)).
|
||||
* Increase connectTimeoutMs IMDS connection timeout to 50ms to avoid failures in CI. [#68653](https://github.com/ClickHouse/ClickHouse/pull/68653) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* CI: Disable SQLLogic job. [#68654](https://github.com/ClickHouse/ClickHouse/pull/68654) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version_date.tsv and changelog after v24.8.1.2684-lts. [#68664](https://github.com/ClickHouse/ClickHouse/pull/68664) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky test test_distributed_replica_max_ignored_errors. [#68665](https://github.com/ClickHouse/ClickHouse/pull/68665) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Improve `02293_http_header_full_summary_without_progress` logging. [#68666](https://github.com/ClickHouse/ClickHouse/pull/68666) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Fix flaky check when all tests are skipped. [#68673](https://github.com/ClickHouse/ClickHouse/pull/68673) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Vector similarity index: make `bf16` the default quantization. [#68678](https://github.com/ClickHouse/ClickHouse/pull/68678) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* 3h is not enough. [#68683](https://github.com/ClickHouse/ClickHouse/pull/68683) ([Max K.](https://github.com/maxknv)).
|
||||
* Un-flake 01278_random_string_utf8. [#68684](https://github.com/ClickHouse/ClickHouse/pull/68684) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* CI: Integration tests timeout to 3h. [#68685](https://github.com/ClickHouse/ClickHouse/pull/68685) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix Upgrade Check: move some settings to 24.9 section. [#68688](https://github.com/ClickHouse/ClickHouse/pull/68688) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Update README.md. [#68690](https://github.com/ClickHouse/ClickHouse/pull/68690) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Use proper ErrorCodes, replace NETWORK_ERROR by HDFS_ERROR. [#68696](https://github.com/ClickHouse/ClickHouse/pull/68696) ([flynn](https://github.com/ucasfl)).
|
||||
* Fix for failing test in CI. [#68701](https://github.com/ClickHouse/ClickHouse/pull/68701) ([Pedro Ferreira](https://github.com/PedroTadim)).
|
||||
* CI: Stress test fix. [#68712](https://github.com/ClickHouse/ClickHouse/pull/68712) ([Max K.](https://github.com/maxknv)).
|
||||
* Speedup test 02150_index_hypothesis_race_long. [#68713](https://github.com/ClickHouse/ClickHouse/pull/68713) ([vdimir](https://github.com/vdimir)).
|
||||
* Turn off fault injection for insert in `01396_inactive_replica_cleanup_nodes_zookeeper`. [#68715](https://github.com/ClickHouse/ClickHouse/pull/68715) ([alesapin](https://github.com/alesapin)).
|
||||
* Update README.md - Meetups update. [#68723](https://github.com/ClickHouse/ClickHouse/pull/68723) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Fix flaky check. [#68725](https://github.com/ClickHouse/ClickHouse/pull/68725) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* fix shutdown for PeriodicLog. [#68728](https://github.com/ClickHouse/ClickHouse/pull/68728) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68729](https://github.com/ClickHouse/ClickHouse/pull/68729) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Bump Replxx to support custom descriptors. [#68730](https://github.com/ClickHouse/ClickHouse/pull/68730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Update version_date.tsv and changelog after v24.6.3.38-stable. [#68732](https://github.com/ClickHouse/ClickHouse/pull/68732) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fixes `00080_show_tables_and_system_tables` that was a bit flaky. [#68734](https://github.com/ClickHouse/ClickHouse/pull/68734) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Update version_date.tsv and changelog after v24.7.3.47-stable. [#68735](https://github.com/ClickHouse/ClickHouse/pull/68735) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky test 00989_parallel_parts_loading. [#68737](https://github.com/ClickHouse/ClickHouse/pull/68737) ([alesapin](https://github.com/alesapin)).
|
||||
* Update README.md. [#68738](https://github.com/ClickHouse/ClickHouse/pull/68738) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Update version_date.tsv and changelog after v24.8.2.3-lts. [#68740](https://github.com/ClickHouse/ClickHouse/pull/68740) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.5.5.41-stable. [#68745](https://github.com/ClickHouse/ClickHouse/pull/68745) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* To make patch release possible from every commit on release branch, package_debug build is required and must not be skipped. [#68750](https://github.com/ClickHouse/ClickHouse/pull/68750) ([Max K.](https://github.com/maxknv)).
|
||||
* Try to disable rerun check if job triggered manually. [#68751](https://github.com/ClickHouse/ClickHouse/pull/68751) ([Max K.](https://github.com/maxknv)).
|
||||
* Fix 2477 timeout. [#68752](https://github.com/ClickHouse/ClickHouse/pull/68752) ([Shichao](https://github.com/jsc0218)).
|
||||
* Update README.md. [#68764](https://github.com/ClickHouse/ClickHouse/pull/68764) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Update version_date.tsv and changelog after v24.5.6.45-stable. [#68766](https://github.com/ClickHouse/ClickHouse/pull/68766) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.6.4.42-stable. [#68767](https://github.com/ClickHouse/ClickHouse/pull/68767) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.7.4.51-stable. [#68768](https://github.com/ClickHouse/ClickHouse/pull/68768) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Split test case and reduce number of random runs to reduce the time necessary to run the test. [#68772](https://github.com/ClickHouse/ClickHouse/pull/68772) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Check setting use_json_alias_for_old_object_type in runtime. [#68793](https://github.com/ClickHouse/ClickHouse/pull/68793) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Make dynamic structure selection more consistent. [#68802](https://github.com/ClickHouse/ClickHouse/pull/68802) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Fix zero copy bug with encrypted disk and UNFREEZE. [#68821](https://github.com/ClickHouse/ClickHouse/pull/68821) ([Joe Lynch](https://github.com/joelynch)).
|
||||
* Speed up functions `lowerUTF8`/`upperUTF8`. [#68822](https://github.com/ClickHouse/ClickHouse/pull/68822) ([李扬](https://github.com/taiyang-li)).
|
||||
* Convert integration test `test_incorrect_datetime_format` to stateless. [#68823](https://github.com/ClickHouse/ClickHouse/pull/68823) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix Function Typo. [#68835](https://github.com/ClickHouse/ClickHouse/pull/68835) ([Shichao](https://github.com/jsc0218)).
|
||||
* Fix test `03228_virtual_column_merge_dist`. [#68841](https://github.com/ClickHouse/ClickHouse/pull/68841) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix test `03221_mutation_analyzer_skip_part`. [#68842](https://github.com/ClickHouse/ClickHouse/pull/68842) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix flaky `02932_analyzer_rewrite_sum_column_and_constant `. [#68843](https://github.com/ClickHouse/ClickHouse/pull/68843) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Update README.md - Add Austin meetup. [#68845](https://github.com/ClickHouse/ClickHouse/pull/68845) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Fix ssl handshake error processing. [#68866](https://github.com/ClickHouse/ClickHouse/pull/68866) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Speedup test 00653_verification_monotonic_data_load. [#68878](https://github.com/ClickHouse/ClickHouse/pull/68878) ([vdimir](https://github.com/vdimir)).
|
||||
* ci: add IPv6 support to `NetworkManager`. [#68900](https://github.com/ClickHouse/ClickHouse/pull/68900) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* ci: add checks for `iptables-nft` in integration test runner. [#68902](https://github.com/ClickHouse/ClickHouse/pull/68902) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Fix `02378_part_log_profile_events` flakiness. [#68917](https://github.com/ClickHouse/ClickHouse/pull/68917) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Make long_parquet* tests less long. [#68918](https://github.com/ClickHouse/ClickHouse/pull/68918) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Fix 01114_database_atomic flakiness. [#68930](https://github.com/ClickHouse/ClickHouse/pull/68930) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* CI: Fix job rerun check. [#68931](https://github.com/ClickHouse/ClickHouse/pull/68931) ([Max K.](https://github.com/maxknv)).
|
||||
* perf tests set cgroups_memory_usage_observer_wait_time to zero. [#68957](https://github.com/ClickHouse/ClickHouse/pull/68957) ([vdimir](https://github.com/vdimir)).
|
||||
* Revert un-quoting `auto` settings values. Fixes [#68748](https://github.com/ClickHouse/ClickHouse/issues/68748). [#68979](https://github.com/ClickHouse/ClickHouse/pull/68979) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Actually fix false "Killed by signal (output files)" in stress_tests.lib. [#68983](https://github.com/ClickHouse/ClickHouse/pull/68983) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* no-tsan in 02735_parquet_encoder. [#68984](https://github.com/ClickHouse/ClickHouse/pull/68984) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Update CHANGELOG.md. [#68994](https://github.com/ClickHouse/ClickHouse/pull/68994) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix test_role & test_keeper_s3_snapshot integration tests. [#69013](https://github.com/ClickHouse/ClickHouse/pull/69013) ([Shankar](https://github.com/shiyer7474)).
|
||||
* fix false leak detect in libfiu. [#69018](https://github.com/ClickHouse/ClickHouse/pull/69018) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Update README.md. [#69041](https://github.com/ClickHouse/ClickHouse/pull/69041) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Update parametric-functions.md. [#69055](https://github.com/ClickHouse/ClickHouse/pull/69055) ([Samuel Warfield](https://github.com/Warfields)).
|
||||
* Disallow `ALTER TABLE ADD VECTOR SIMILARITY INDEX` if corresponding setting is not enabled. [#69065](https://github.com/ClickHouse/ClickHouse/pull/69065) ([flynn](https://github.com/ucasfl)).
|
||||
* Remove stale moving parts without zookeeper. [#69075](https://github.com/ClickHouse/ClickHouse/pull/69075) ([Kirill](https://github.com/kirillgarbar)).
|
||||
* Update README.md. [#69077](https://github.com/ClickHouse/ClickHouse/pull/69077) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Fix typo in ActionsDag. [#69086](https://github.com/ClickHouse/ClickHouse/pull/69086) ([LiuNeng](https://github.com/liuneng1994)).
|
||||
* Follow-up for https://github.com/ClickHouse/ClickHouse/pull/68332. [#69099](https://github.com/ClickHouse/ClickHouse/pull/69099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix subnet in docker_compose_net.yml. [#69121](https://github.com/ClickHouse/ClickHouse/pull/69121) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Fix: expression description in plan after lift up union optimization. [#69124](https://github.com/ClickHouse/ClickHouse/pull/69124) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix locking in UserDefinedSQLObjectsZooKeeperStorage.cpp. [#69132](https://github.com/ClickHouse/ClickHouse/pull/69132) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Do not use docker pause for Kerberos KDC container in integration tests. [#69136](https://github.com/ClickHouse/ClickHouse/pull/69136) ([Ilya Golshtein](https://github.com/ilejn)).
|
||||
* Don't create Object type if use_json_alias_for_old_object_type=1 but allow_experimental_object_type=0. [#69150](https://github.com/ClickHouse/ClickHouse/pull/69150) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Use QueryPlan for merge. [#69167](https://github.com/ClickHouse/ClickHouse/pull/69167) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Update version_date.tsv and changelog after v24.3.10.33-lts. [#69177](https://github.com/ClickHouse/ClickHouse/pull/69177) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* CI: Rerun check: do not check if manual rerun. [#69181](https://github.com/ClickHouse/ClickHouse/pull/69181) ([Max K.](https://github.com/maxknv)).
|
||||
* Update version_date.tsv and changelog after v24.5.7.31-stable. [#69182](https://github.com/ClickHouse/ClickHouse/pull/69182) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.6.5.30-stable. [#69184](https://github.com/ClickHouse/ClickHouse/pull/69184) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix flaky 02915_move_partition_inactive_replica. [#69190](https://github.com/ClickHouse/ClickHouse/pull/69190) ([alesapin](https://github.com/alesapin)).
|
||||
* Update version_date.tsv and changelog after v24.8.3.59-lts. [#69191](https://github.com/ClickHouse/ClickHouse/pull/69191) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Disable memory test with sanitizer. [#69193](https://github.com/ClickHouse/ClickHouse/pull/69193) ([alesapin](https://github.com/alesapin)).
|
||||
* Disable perf-like test with sanitizers. [#69194](https://github.com/ClickHouse/ClickHouse/pull/69194) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix jepsen for aarch64. [#69199](https://github.com/ClickHouse/ClickHouse/pull/69199) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Test for parallel replicas, reverse in order reading mode. [#69202](https://github.com/ClickHouse/ClickHouse/pull/69202) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Collect sanitizer report from client to `client_log`. [#69204](https://github.com/ClickHouse/ClickHouse/pull/69204) ([vdimir](https://github.com/vdimir)).
|
||||
* Fix flaky 00157_cache_dictionary. [#69205](https://github.com/ClickHouse/ClickHouse/pull/69205) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Hide Settings implementation. [#69213](https://github.com/ClickHouse/ClickHouse/pull/69213) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix progress bar when reading from Memory tables. [#69234](https://github.com/ClickHouse/ClickHouse/pull/69234) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* CMake: Update ICU build description. [#69240](https://github.com/ClickHouse/ClickHouse/pull/69240) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add testcase for ANN index usage with subquery. [#69241](https://github.com/ClickHouse/ClickHouse/pull/69241) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backports should read tags from its repo only. [#69252](https://github.com/ClickHouse/ClickHouse/pull/69252) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* 01114_database_atomic: Increase time frames to reduce flakiness. [#69255](https://github.com/ClickHouse/ClickHouse/pull/69255) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix dropping of file cache in CHECK query in case of enabled transactions. [#69256](https://github.com/ClickHouse/ClickHouse/pull/69256) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* CI: Merge stress and func runners type. [#69257](https://github.com/ClickHouse/ClickHouse/pull/69257) ([Max K.](https://github.com/maxknv)).
|
||||
* Make infrastructure related scripts private. [#69260](https://github.com/ClickHouse/ClickHouse/pull/69260) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix: Not-ready Set with parallel replicas. [#69264](https://github.com/ClickHouse/ClickHouse/pull/69264) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Revert "CREATE TABLE AS copy PRIMARY KEY, ORDER BY, and similar clauses.". [#69268](https://github.com/ClickHouse/ClickHouse/pull/69268) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Use CANNOT_DECOMPRESS error code for the compressed external data in CompressedReadBufferBase for checksum validation and decompression. [#69269](https://github.com/ClickHouse/ClickHouse/pull/69269) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Fix the error in loading client suggestions and select from `system.function` when `allow_experimental_nlp_functions` setting is enabled and no lemmatizers are specified in the configuration. [#69271](https://github.com/ClickHouse/ClickHouse/pull/69271) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Improve logical error trace for TryResult. [#69297](https://github.com/ClickHouse/ClickHouse/pull/69297) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Just a small refactoring. [#69298](https://github.com/ClickHouse/ClickHouse/pull/69298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Don't run 01287_max_execution_speed with slow builds. [#69299](https://github.com/ClickHouse/ClickHouse/pull/69299) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Add checks against segfault in DeltaLakeMetadata. [#69305](https://github.com/ClickHouse/ClickHouse/pull/69305) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Update StackTrace.cpp. [#69306](https://github.com/ClickHouse/ClickHouse/pull/69306) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Bump libarchive from v3.7.0 to v3.7.1. [#69318](https://github.com/ClickHouse/ClickHouse/pull/69318) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump curl to v8.9.1. [#69321](https://github.com/ClickHouse/ClickHouse/pull/69321) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* These tests take long time to execute with slow builds (either 700 or 400 seconds). [#69322](https://github.com/ClickHouse/ClickHouse/pull/69322) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Bump grpc to v1.59.5. [#69323](https://github.com/ClickHouse/ClickHouse/pull/69323) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Speed up some Kafka tests with multiprocessing. [#69324](https://github.com/ClickHouse/ClickHouse/pull/69324) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Minor follow-up for [#66933](https://github.com/ClickHouse/ClickHouse/issues/66933). [#69326](https://github.com/ClickHouse/ClickHouse/pull/69326) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Randomize integration tests settings. [#69328](https://github.com/ClickHouse/ClickHouse/pull/69328) ([vdimir](https://github.com/vdimir)).
|
||||
* Update version_date.tsv and changelog after v24.3.11.7-lts. [#69330](https://github.com/ClickHouse/ClickHouse/pull/69330) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.5.8.10-stable. [#69333](https://github.com/ClickHouse/ClickHouse/pull/69333) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Add CITATION.cff. [#69334](https://github.com/ClickHouse/ClickHouse/pull/69334) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelog after v24.6.6.6-stable. [#69335](https://github.com/ClickHouse/ClickHouse/pull/69335) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Fix a leak of a few bytes in function `lower/upperUTF8`. [#69336](https://github.com/ClickHouse/ClickHouse/pull/69336) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update version_date.tsv and changelog after v24.7.6.8-stable. [#69337](https://github.com/ClickHouse/ClickHouse/pull/69337) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Update version_date.tsv and changelog after v24.8.4.13-lts. [#69340](https://github.com/ClickHouse/ClickHouse/pull/69340) ([robot-clickhouse](https://github.com/robot-clickhouse)).
|
||||
* Bump OpenSSL to v3.2.3. [#69341](https://github.com/ClickHouse/ClickHouse/pull/69341) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. V. [#69344](https://github.com/ClickHouse/ClickHouse/pull/69344) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump libuv to v1.48.0. [#69345](https://github.com/ClickHouse/ClickHouse/pull/69345) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Update README.md - Meetups. [#69359](https://github.com/ClickHouse/ClickHouse/pull/69359) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* Remove obsolete `--multiquery` parameter (follow-up to [#63898](https://github.com/ClickHouse/ClickHouse/issues/63898)), pt. VI. [#69361](https://github.com/ClickHouse/ClickHouse/pull/69361) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Bump libarchive to v3.7.4. [#69367](https://github.com/ClickHouse/ClickHouse/pull/69367) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Rename `count_min` statistics to `countmin`. [#69377](https://github.com/ClickHouse/ClickHouse/pull/69377) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Refactor temporary file usage in MergeTask with QueryPlan. [#69383](https://github.com/ClickHouse/ClickHouse/pull/69383) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix type mismatch. [#69385](https://github.com/ClickHouse/ClickHouse/pull/69385) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix 24.8 setting compatibility `rows_before_aggregation`. [#69394](https://github.com/ClickHouse/ClickHouse/pull/69394) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Support more oss endpoints. [#69400](https://github.com/ClickHouse/ClickHouse/pull/69400) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix 01603_read_with_backoff_bug. [#69413](https://github.com/ClickHouse/ClickHouse/pull/69413) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* See if it would address strange issue with `Resource temporarily unavailable`. [#69416](https://github.com/ClickHouse/ClickHouse/pull/69416) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Added a server setting `database_replicated_allow_detach_permanently` that disallows `DETACH TABLE PERMANENTLY` queries in Replicated databases (allowed by default). [#69422](https://github.com/ClickHouse/ClickHouse/pull/69422) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* See https://s3.amazonaws.com/clickhouse-test-reports/69093/d9b1fb7a7cb0813dd12d8f73423662e02a458056/fast_test.html. [#69430](https://github.com/ClickHouse/ClickHouse/pull/69430) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix test_disks_app_func. [#69449](https://github.com/ClickHouse/ClickHouse/pull/69449) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fixed consistency for `groupConcat` with `arrayStringConcat` in case of `FixedString` as argument. [#69455](https://github.com/ClickHouse/ClickHouse/pull/69455) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* The fix from [#69416](https://github.com/ClickHouse/ClickHouse/issues/69416) was incomplete. Reduce the size of printed statement to 4k max. [#69483](https://github.com/ClickHouse/ClickHouse/pull/69483) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* CMake: Add comment about ICU data files. [#69511](https://github.com/ClickHouse/ClickHouse/pull/69511) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* S3Queue: small refactoring. [#69513](https://github.com/ClickHouse/ClickHouse/pull/69513) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backport MergeTask changes from private to minimize merge conflicts. [#69525](https://github.com/ClickHouse/ClickHouse/pull/69525) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix getting the latest synced commit for long PRs. [#69538](https://github.com/ClickHouse/ClickHouse/pull/69538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* When trying to ATTACH ReplicatedMergeTree tables that were previously PERMANENTLY DETACHED, we will fail due to explicitly specified parameters. Reference https://github.com/ClickHouse/ClickHouse/pull/66104. [#69539](https://github.com/ClickHouse/ClickHouse/pull/69539) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix ci upgrade check failure for `join_to_sort` settings in pr https://github.com/ClickHouse/ClickHouse/pull/60341. [#69554](https://github.com/ClickHouse/ClickHouse/pull/69554) ([kevinyhzou](https://github.com/KevinyhZou)).
|
||||
* Add function `kill_ci_runner`. Kill runner when pre-pull failed. [#69557](https://github.com/ClickHouse/ClickHouse/pull/69557) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Replace libpq code dump by postgresql fork + bump to v14.3. [#69564](https://github.com/ClickHouse/ClickHouse/pull/69564) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Mask azure connection string sensitive info. [#69570](https://github.com/ClickHouse/ClickHouse/pull/69570) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Don't leave an empty znode when replicated table is dropped. [#69574](https://github.com/ClickHouse/ClickHouse/pull/69574) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Enable removerecursive in ci. [#69578](https://github.com/ClickHouse/ClickHouse/pull/69578) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* Bump libpqxx to v7.7.5. [#69580](https://github.com/ClickHouse/ClickHouse/pull/69580) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Remove superfluous `--multiquery/-n`, pt. V. [#69581](https://github.com/ClickHouse/ClickHouse/pull/69581) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* A better fix for [#67476](https://github.com/ClickHouse/ClickHouse/issues/67476). [#69583](https://github.com/ClickHouse/ClickHouse/pull/69583) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add more contexts to the debug action and use it broadly. [#69599](https://github.com/ClickHouse/ClickHouse/pull/69599) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Try to fix data race in `WriteBufferFromHTTPServerResponse`. [#69601](https://github.com/ClickHouse/ClickHouse/pull/69601) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Remove explicit announce from local replica in ReadFromMergeTree. [#69602](https://github.com/ClickHouse/ClickHouse/pull/69602) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Unification of FunctionSecretArgumentsFinder. [#69615](https://github.com/ClickHouse/ClickHouse/pull/69615) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Reorder some tests (follow-up to [#69514](https://github.com/ClickHouse/ClickHouse/issues/69514)). [#69626](https://github.com/ClickHouse/ClickHouse/pull/69626) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
|
||||
* Fix invalid `clickhouse-format` invocation in tests. [#69642](https://github.com/ClickHouse/ClickHouse/pull/69642) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Try fix `02447_drop_database_replica`. [#69655](https://github.com/ClickHouse/ClickHouse/pull/69655) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* S3Queue small refactoring. [#69672](https://github.com/ClickHouse/ClickHouse/pull/69672) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Update assert. [#69673](https://github.com/ClickHouse/ClickHouse/pull/69673) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Bump libpq from v14.3 to v15.8. [#69674](https://github.com/ClickHouse/ClickHouse/pull/69674) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Try fix asserts failure in `HashJoin`. [#69682](https://github.com/ClickHouse/ClickHouse/pull/69682) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Prohibit `ALTER TABLE ... ADD INDEX ... TYPE` inverted if setting = 0. [#69684](https://github.com/ClickHouse/ClickHouse/pull/69684) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fixed 2 bugs in Remove Recursive implementation:. [#69690](https://github.com/ClickHouse/ClickHouse/pull/69690) ([Mikhail Artemenko](https://github.com/Michicosun)).
|
||||
* New profile events for parallel replicas. [#69706](https://github.com/ClickHouse/ClickHouse/pull/69706) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Update README.md - Meetups. [#69714](https://github.com/ClickHouse/ClickHouse/pull/69714) ([Tanya Bragin](https://github.com/tbragin)).
|
||||
* added some edge cases for printf tests. [#69737](https://github.com/ClickHouse/ClickHouse/pull/69737) ([max-vostrikov](https://github.com/max-vostrikov)).
|
||||
* Bump libpq to v16.4. [#69741](https://github.com/ClickHouse/ClickHouse/pull/69741) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix parallel replicas protocol after [#68424](https://github.com/ClickHouse/ClickHouse/issues/68424). [#69744](https://github.com/ClickHouse/ClickHouse/pull/69744) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* CI: Include PostgreSQL in sparse checkout script. [#69746](https://github.com/ClickHouse/ClickHouse/pull/69746) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Get rid of the import cycle when env_helper can't be imported from `ci_*.py` files. Now, `env_helper` won't import download helper or basically anything from the `ci` module. The report is the best place for the GH job info since we use it mostly there. [#69750](https://github.com/ClickHouse/ClickHouse/pull/69750) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Save CREATE QUERY with the KeeperMap engine with evaluated parameters. It is important because, for example, we can use the `currentDatabase()` function in the KeeperMap `zk_root_path` argument, and we want it to point to the same path even if we move this table to another database. [#69751](https://github.com/ClickHouse/ClickHouse/pull/69751) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Fix native macOS build. [#69767](https://github.com/ClickHouse/ClickHouse/pull/69767) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Backported in [#69974](https://github.com/ClickHouse/ClickHouse/issues/69974): S3Queue: support having deprecated settings to not fail server startup. [#69769](https://github.com/ClickHouse/ClickHouse/pull/69769) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Limit fast tests to 30 seconds. [#69781](https://github.com/ClickHouse/ClickHouse/pull/69781) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69824](https://github.com/ClickHouse/ClickHouse/issues/69824): Allow cyrillic characters in generated contributor names. [#69820](https://github.com/ClickHouse/ClickHouse/pull/69820) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#69912](https://github.com/ClickHouse/ClickHouse/issues/69912): Add Proj Obsolete Setting. [#69883](https://github.com/ClickHouse/ClickHouse/pull/69883) ([Shichao](https://github.com/jsc0218)).
|
||||
* Backported in [#69899](https://github.com/ClickHouse/ClickHouse/issues/69899): Revert "Merge pull request [#69032](https://github.com/ClickHouse/ClickHouse/issues/69032) from alexon1234/include_real_time_execution_in_http_header". [#69885](https://github.com/ClickHouse/ClickHouse/pull/69885) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#69931](https://github.com/ClickHouse/ClickHouse/issues/69931): RIPE is an acronym and thus should be capital. RIPE stands for **R**ACE **I**ntegrity **P**rimitives **E**valuation and RACE stands for **R**esearch and Development in **A**dvanced **C**ommunications **T**echnologies in **E**urope. [#69901](https://github.com/ClickHouse/ClickHouse/pull/69901) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Backported in [#70034](https://github.com/ClickHouse/ClickHouse/issues/70034): Revert "Add RIPEMD160 function". [#70005](https://github.com/ClickHouse/ClickHouse/pull/70005) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
|
||||
33
docs/changelogs/v24.9.2.42-stable.md
Normal file
33
docs/changelogs/v24.9.2.42-stable.md
Normal file
@ -0,0 +1,33 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.9.2.42-stable (de7c791a2ea) FIXME as compared to v24.9.1.3278-stable (6d058d82a8e)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#70091](https://github.com/ClickHouse/ClickHouse/issues/70091): Add `show_create_query_identifier_quoting_rule` to define identifier quoting behavior of the show create query result. Possible values: - `user_display`: When the identifiers is a keyword. - `when_necessary`: When the identifiers is one of `{"distinct", "all", "table"}`, or it can cause ambiguity: column names, dictionary attribute names. - `always`: Always quote identifiers. [#69448](https://github.com/ClickHouse/ClickHouse/pull/69448) ([tuanpach](https://github.com/tuanpach)).
|
||||
* Backported in [#70100](https://github.com/ClickHouse/ClickHouse/issues/70100): Follow-up to https://github.com/ClickHouse/ClickHouse/pull/69346 Point 4 described there will work now as well:. [#69563](https://github.com/ClickHouse/ClickHouse/pull/69563) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#70048](https://github.com/ClickHouse/ClickHouse/issues/70048): Add new column readonly_duration to the system.replicas table. Needed to be able to distinguish actual readonly replicas from sentinel ones in alerts. [#69871](https://github.com/ClickHouse/ClickHouse/pull/69871) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#70193](https://github.com/ClickHouse/ClickHouse/issues/70193): Fix crash when executing `create view t as (with recursive 42 as ttt select ttt);`. [#69676](https://github.com/ClickHouse/ClickHouse/pull/69676) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Backported in [#70083](https://github.com/ClickHouse/ClickHouse/issues/70083): Closes [#69752](https://github.com/ClickHouse/ClickHouse/issues/69752). [#69985](https://github.com/ClickHouse/ClickHouse/pull/69985) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#70070](https://github.com/ClickHouse/ClickHouse/issues/70070): Fixes `Block structure mismatch` for queries with nested views and `WHERE` condition. Fixes [#66209](https://github.com/ClickHouse/ClickHouse/issues/66209). [#70054](https://github.com/ClickHouse/ClickHouse/pull/70054) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#70168](https://github.com/ClickHouse/ClickHouse/issues/70168): Fix wrong LOGICAL_ERROR when replacing literals in ranges. [#70122](https://github.com/ClickHouse/ClickHouse/pull/70122) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#70238](https://github.com/ClickHouse/ClickHouse/issues/70238): Check for Nullable(Nothing) type during ALTER TABLE MODIFY COLUMN/QUERY to prevent tables with such data type. [#70123](https://github.com/ClickHouse/ClickHouse/pull/70123) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70205](https://github.com/ClickHouse/ClickHouse/issues/70205): Fix wrong result with skipping index. [#70127](https://github.com/ClickHouse/ClickHouse/pull/70127) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#70185](https://github.com/ClickHouse/ClickHouse/issues/70185): Fix data race in ColumnObject/ColumnTuple decompress method that could lead to heap use after free. [#70137](https://github.com/ClickHouse/ClickHouse/pull/70137) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70253](https://github.com/ClickHouse/ClickHouse/issues/70253): Fix possible hung in ALTER COLUMN with Dynamic type. [#70144](https://github.com/ClickHouse/ClickHouse/pull/70144) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70230](https://github.com/ClickHouse/ClickHouse/issues/70230): Use correct `max_types` parameter during Dynamic type creation for JSON subcolumn. [#70147](https://github.com/ClickHouse/ClickHouse/pull/70147) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#70217](https://github.com/ClickHouse/ClickHouse/issues/70217): Fix the password being displayed in `system.query_log` for users with bcrypt password authentication method. [#70148](https://github.com/ClickHouse/ClickHouse/pull/70148) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Backported in [#70267](https://github.com/ClickHouse/ClickHouse/issues/70267): Respect setting allow_simdjson in JSON type parser. [#70218](https://github.com/ClickHouse/ClickHouse/pull/70218) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#70052](https://github.com/ClickHouse/ClickHouse/issues/70052): Improve stateless test runner. [#69864](https://github.com/ClickHouse/ClickHouse/pull/69864) ([Alexey Katsman](https://github.com/alexkats)).
|
||||
* Backported in [#70284](https://github.com/ClickHouse/ClickHouse/issues/70284): Improve pipdeptree generator for docker images. - Update requirements.txt for the integration tests runner container - Remove some small dependencies, improve `helpers/retry_decorator.py` - Upgrade docker-compose from EOL version 1 to version 2. [#70146](https://github.com/ClickHouse/ClickHouse/pull/70146) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#70261](https://github.com/ClickHouse/ClickHouse/issues/70261): Update test_storage_s3_queue/test.py. [#70159](https://github.com/ClickHouse/ClickHouse/pull/70159) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
@ -232,7 +232,7 @@ Below you can find some quick links which may be useful when writing code for Cl
|
||||
- [The code style guide](https://clickhouse.com/docs/en/development/style/).
|
||||
- [Adding third-party libraries](https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries)
|
||||
- [Writing tests](https://clickhouse.com/docs/en/development/tests/)
|
||||
- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest)
|
||||
- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22)
|
||||
|
||||
## Writing Documentation {#writing-documentation}
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ description: Analyzing Stack Overflow data with ClickHouse
|
||||
|
||||
# Analyzing Stack Overflow data with ClickHouse
|
||||
|
||||
This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow.
|
||||
This dataset contains every `Posts`, `Users`, `Votes`, `Comments`, `Badges`, `PostHistory`, and `PostLinks` that has occurred on Stack Overflow.
|
||||
|
||||
Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months.
|
||||
|
||||
@ -159,7 +159,7 @@ INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentatio
|
||||
0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.)
|
||||
```
|
||||
|
||||
### `PostLinks`
|
||||
### PostLinks
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.postlinks
|
||||
@ -178,7 +178,7 @@ INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documenta
|
||||
0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.)
|
||||
```
|
||||
|
||||
### `PostHistory`
|
||||
### PostHistory
|
||||
|
||||
```sql
|
||||
CREATE TABLE stackoverflow.posthistory
|
||||
|
||||
@ -195,6 +195,9 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--print-profile-events` – Print `ProfileEvents` packets.
|
||||
- `--profile-events-delay-ms` – Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet).
|
||||
- `--jwt` – If specified, enables authorization via JSON Web Token. Server JWT authorization is available only in ClickHouse Cloud.
|
||||
- `--progress` – Print progress of query execution. Possible values: 'tty|on|1|true|yes' - outputs to TTY in interactive mode; 'err' - outputs to STDERR non-interactive mode; 'off|0|false|no' - disables the progress printing. Default: TTY in interactive mode, disabled in non-interactive.
|
||||
- `--progress-table` – Print a progress table with changing metrics during query execution. Possible values: 'tty|on|1|true|yes' - outputs to TTY in interactive mode; 'err' - outputs to STDERR non-interactive mode; 'off|0|false|no' - disables the progress table. Default: TTY in interactive mode, disabled in non-interactive.
|
||||
- `--enable-progress-table-toggle` – Enable toggling of the progress table by pressing the control key (Space). Only applicable in interactive mode with the progress table printing enabled. Default: 'true'.
|
||||
|
||||
Instead of `--host`, `--port`, `--user` and `--password` options, ClickHouse client also supports connection strings (see next section).
|
||||
|
||||
|
||||
@ -1598,10 +1598,6 @@ the columns from input data will be mapped to the columns from the table by thei
|
||||
Otherwise, the first row will be skipped.
|
||||
If setting [input_format_with_types_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_types_use_header) is set to 1,
|
||||
the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
|
||||
If setting [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) is set to 1,
|
||||
the types in header will be written using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format.
|
||||
If setting [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) is set to 1,
|
||||
the types in header will be read using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format.
|
||||
:::
|
||||
|
||||
## RowBinaryWithDefaults {#rowbinarywithdefaults}
|
||||
@ -1624,6 +1620,10 @@ For column `y` data starts with byte `00` that indicates that column has actual
|
||||
## RowBinary format settings {#row-binary-format-settings}
|
||||
|
||||
- [format_binary_max_string_size](/docs/en/operations/settings/settings-formats.md/#format_binary_max_string_size) - The maximum allowed size for String in RowBinary format. Default value - `1GiB`.
|
||||
- [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) - Allows to write types in header using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format. Default value - `false`.
|
||||
- [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) - Allows to read types in header using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format. Default value - `false`.
|
||||
- [output_format_binary_write_json_as_string](/docs/en/operations/settings/settings-formats.md/#output_format_binary_write_json_as_string) - Allows to write values of [JSON](/docs/en/sql-reference/data-types/newjson.md) data type as JSON [String](/docs/en/sql-reference/data-types/string.md) values in RowBinary output format. Default value - `false`.
|
||||
- [input_format_binary_read_json_as_string](/docs/en/operations/settings/settings-formats.md/#input_format_binary_read_json_as_string) - Allows to read values of [JSON](/docs/en/sql-reference/data-types/newjson.md) data type as JSON [String](/docs/en/sql-reference/data-types/string.md) values in RowBinary input format. Default value - `false`.
|
||||
|
||||
## Values {#data-format-values}
|
||||
|
||||
|
||||
@ -1057,11 +1057,39 @@ Default value: throw
|
||||
|
||||
## deduplicate_merge_projection_mode
|
||||
|
||||
Whether to allow create projection for the table with non-classic MergeTree, that is not (Replicated, Shared) MergeTree. If allowed, what is the action when merge projections, either drop or rebuild. So classic MergeTree would ignore this setting.
|
||||
Whether to allow create projection for the table with non-classic MergeTree, that is not (Replicated, Shared) MergeTree. Ignore option is purely for compatibility which might result in incorrect answer. Otherwise, if allowed, what is the action when merge projections, either drop or rebuild. So classic MergeTree would ignore this setting.
|
||||
It also controls `OPTIMIZE DEDUPLICATE` as well, but has effect on all MergeTree family members. Similar to the option `lightweight_mutation_projection_mode`, it is also part level.
|
||||
|
||||
Possible values:
|
||||
|
||||
- throw, drop, rebuild
|
||||
- ignore, throw, drop, rebuild
|
||||
|
||||
Default value: throw
|
||||
Default value: throw
|
||||
|
||||
## min_free_disk_bytes_to_perform_insert
|
||||
|
||||
The minimum number of bytes that should be free in disk space in order to insert data. If the number of available free bytes is less than `min_free_disk_bytes_to_throw_insert` then an exception is thrown and the insert is not executed. Note that this setting:
|
||||
- takes into account the `keep_free_space_bytes` setting.
|
||||
- does not take into account the amount of data that will be written by the `INSERT` operation.
|
||||
- is only checked if a positive (non-zero) number of bytes is specified
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 0 bytes.
|
||||
|
||||
## min_free_disk_ratio_to_perform_insert
|
||||
|
||||
The minimum free to total disk space ratio to perform an `INSERT`. Must be a floating point value between 0 and 1. Note that this setting:
|
||||
- takes into account the `keep_free_space_bytes` setting.
|
||||
- does not take into account the amount of data that will be written by the `INSERT` operation.
|
||||
- is only checked if a positive (non-zero) ratio is specified
|
||||
|
||||
Possible values:
|
||||
|
||||
- Float, 0.0 - 1.0
|
||||
|
||||
Default value: 0.0
|
||||
|
||||
Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
|
||||
|
||||
@ -49,6 +49,18 @@ Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHo
|
||||
|
||||
See also the description of [max_memory_usage](#settings_max_memory_usage).
|
||||
|
||||
For example if you want to set `max_memory_usage_for_user` to 1000 bytes for a user named `clickhouse_read`, you can use the statement
|
||||
|
||||
``` sql
|
||||
ALTER USER clickhouse_read SETTINGS max_memory_usage_for_user = 1000;
|
||||
```
|
||||
|
||||
You can verify it worked by logging out of your client, logging back in, then use the `getSetting` function:
|
||||
|
||||
```sql
|
||||
SELECT getSetting('max_memory_usage_for_user');
|
||||
```
|
||||
|
||||
## max_rows_to_read {#max-rows-to-read}
|
||||
|
||||
The following restrictions can be checked on each block (instead of on each row). That is, the restrictions can be broken a little.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -6,7 +6,7 @@ sidebar_label: User Settings
|
||||
|
||||
# Users and Roles Settings
|
||||
|
||||
The `users` section of the `user.xml` configuration file contains user settings.
|
||||
The `users` section of the `users.xml` configuration file contains user settings.
|
||||
|
||||
:::note
|
||||
ClickHouse also supports [SQL-driven workflow](../../guides/sre/user-management/index.md#access-control) for managing users. We recommend using it.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -10,21 +10,21 @@ Columns:
|
||||
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
|
||||
- `view` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid (Atomic database).
|
||||
- `status` ([String](../../sql-reference/data-types/string.md)) — Current state of the refresh.
|
||||
- `last_refresh_result` ([String](../../sql-reference/data-types/string.md)) — Outcome of the latest refresh attempt.
|
||||
- `last_refresh_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the last refresh attempt. `NULL` if no refresh attempts happened since server startup or table creation.
|
||||
- `last_success_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the last successful refresh. `NULL` if no successful refreshes happened since server startup or table creation.
|
||||
- `duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md)) — How long the last refresh attempt took.
|
||||
- `next_refresh_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time at which the next refresh is scheduled to start.
|
||||
- `remaining_dependencies` ([Array(String)](../../sql-reference/data-types/array.md)) — If the view has [refresh dependencies](../../sql-reference/statements/create/view.md#refresh-dependencies), this array contains the subset of those dependencies that are not satisfied for the current refresh yet. If `status = 'WaitingForDependencies'`, a refresh is ready to start as soon as these dependencies are fulfilled.
|
||||
- `exception` ([String](../../sql-reference/data-types/string.md)) — if `last_refresh_result = 'Error'`, i.e. the last refresh attempt failed, this column contains the corresponding error message and stack trace.
|
||||
- `retry` ([UInt64](../../sql-reference/data-types/int-uint.md)) — If nonzero, the current or next refresh is a retry (see `refresh_retries` refresh setting), and `retry` is the 1-based index of that retry.
|
||||
- `refresh_count` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of successful refreshes since last server restart or table creation.
|
||||
- `progress` ([Float64](../../sql-reference/data-types/float.md)) — Progress of the current refresh, between 0 and 1.
|
||||
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of rows read by the current refresh so far.
|
||||
- `total_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Estimated total number of rows that need to be read by the current refresh.
|
||||
|
||||
(There are additional columns related to current refresh progress, but they are currently unreliable.)
|
||||
- `last_success_time` ([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md))) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation.
|
||||
- `last_success_duration_ms` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — How long the latest refresh took.
|
||||
- `last_refresh_time` ([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md))) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation.
|
||||
- `last_refresh_replica` ([String](../../sql-reference/data-types/string.md)) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt.
|
||||
- `next_refresh_time` ([Nullable](../../sql-reference/data-types/nullable.md)([DateTime](../../sql-reference/data-types/datetime.md))) — Time at which the next refresh is scheduled to start, if status = Scheduled.
|
||||
- `exception` ([String](../../sql-reference/data-types/string.md)) — Error message from previous attempt if it failed.
|
||||
- `retry` ([UInt64](../../sql-reference/data-types/int-uint.md)) — How many failed attempts there were so far, for the current refresh.
|
||||
- `progress` ([Float64](../../sql-reference/data-types/float.md)) — Progress of the current refresh, between 0 and 1. Not available if status is `RunningOnAnotherReplica`.
|
||||
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of rows read by the current refresh so far. Not available if status is `RunningOnAnotherReplica`.
|
||||
- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of bytes read during the current refresh. Not available if status is `RunningOnAnotherReplica`.
|
||||
- `total_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Estimated total number of rows that need to be read by the current refresh. Not available if status is `RunningOnAnotherReplica`.
|
||||
- `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of rows written during the current refresh. Not available if status is `RunningOnAnotherReplica`.
|
||||
- `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number rof bytes written during the current refresh. Not available if status is `RunningOnAnotherReplica`.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -19,9 +19,7 @@ avgWeighted(x, weight)
|
||||
- `weight` — Weights of the values.
|
||||
|
||||
`x` and `weight` must both be
|
||||
[Integer](../../../sql-reference/data-types/int-uint.md),
|
||||
[floating-point](../../../sql-reference/data-types/float.md), or
|
||||
[Decimal](../../../sql-reference/data-types/decimal.md),
|
||||
[Integer](../../../sql-reference/data-types/int-uint.md) or [floating-point](../../../sql-reference/data-types/float.md),
|
||||
but may have different types.
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -297,99 +297,257 @@ $$)
|
||||
└───────────────┴────────────────┴───────────────┴──────┴───────┴────────────┴─────────┘
|
||||
```
|
||||
|
||||
## Comparing values of Dynamic type
|
||||
## Using Dynamic type in functions
|
||||
|
||||
Values of `Dynamic` types are compared similar to values of `Variant` type:
|
||||
Most of the functions support arguments with type `Dynamic`. In this case the function is executed separately on each internal data type stored inside `Dynamic` column.
|
||||
When the result type of the function depends on the arguments types, the result of such function executed with `Dynamic` arguments will be `Dynamic`. When the result type of the function doesn't depend on the arguments types - the result will be `Nullable(T)` where `T` the usual result type of this function.
|
||||
|
||||
Examples:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
INSERT INTO test VALUES (NULL), (1::Int8), (2::Int16), (3::Int32), (4::Int64);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d────┬─dynamicType(d)─┐
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
│ 1 │ Int8 │
|
||||
│ 2 │ Int16 │
|
||||
│ 3 │ Int32 │
|
||||
│ 4 │ Int64 │
|
||||
└──────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
|
||||
│ 1 │ 2 │ Dynamic │ Int16 │
|
||||
│ 2 │ 3 │ Dynamic │ Int32 │
|
||||
│ 3 │ 4 │ Dynamic │ Int64 │
|
||||
│ 4 │ 5 │ Dynamic │ Int64 │
|
||||
└──────┴──────┴─────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, d + d AS res, toTypeName(res), dynamicType(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
|
||||
│ 1 │ 2 │ Dynamic │ Int16 │
|
||||
│ 2 │ 4 │ Dynamic │ Int32 │
|
||||
│ 3 │ 6 │ Dynamic │ Int64 │
|
||||
│ 4 │ 8 │ Dynamic │ Int64 │
|
||||
└──────┴──────┴─────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, d < 3 AS res, toTypeName(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d────┬──res─┬─toTypeName(res)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(UInt8) │
|
||||
│ 1 │ 1 │ Nullable(UInt8) │
|
||||
│ 2 │ 1 │ Nullable(UInt8) │
|
||||
│ 3 │ 0 │ Nullable(UInt8) │
|
||||
│ 4 │ 0 │ Nullable(UInt8) │
|
||||
└──────┴──────┴─────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, exp2(d) AS res, toTypeName(res) FROM test;
|
||||
```
|
||||
|
||||
```sql
|
||||
┌─d────┬──res─┬─toTypeName(res)───┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(Float64) │
|
||||
│ 1 │ 2 │ Nullable(Float64) │
|
||||
│ 2 │ 4 │ Nullable(Float64) │
|
||||
│ 3 │ 8 │ Nullable(Float64) │
|
||||
│ 4 │ 16 │ Nullable(Float64) │
|
||||
└──────┴──────┴───────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
TRUNCATE TABLE test;
|
||||
INSERT INTO test VALUES (NULL), ('str_1'), ('str_2');
|
||||
SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬─dynamicType(d)─┐
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
│ str_1 │ String │
|
||||
│ str_2 │ String │
|
||||
└───────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, upper(d) AS res, toTypeName(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬─res───┬─toTypeName(res)──┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(String) │
|
||||
│ str_1 │ STR_1 │ Nullable(String) │
|
||||
│ str_2 │ STR_2 │ Nullable(String) │
|
||||
└───────┴───────┴──────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, extract(d, '([0-3])') AS res, toTypeName(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬─res──┬─toTypeName(res)──┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(String) │
|
||||
│ str_1 │ 1 │ Nullable(String) │
|
||||
│ str_2 │ 2 │ Nullable(String) │
|
||||
└───────┴──────┴──────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
TRUNCATE TABLE test;
|
||||
INSERT INTO test VALUES (NULL), ([1, 2]), ([3, 4]);
|
||||
SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬─dynamicType(d)─┐
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
│ [1,2] │ Array(Int64) │
|
||||
│ [3,4] │ Array(Int64) │
|
||||
└───────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, d[1] AS res, toTypeName(res), dynamicType(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬─res──┬─toTypeName(res)─┬─dynamicType(res)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dynamic │ None │
|
||||
│ [1,2] │ 1 │ Dynamic │ Int64 │
|
||||
│ [3,4] │ 3 │ Dynamic │ Int64 │
|
||||
└───────┴──────┴─────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
If function cannot be executed on some type inside `Dynamic` column, the exception will be thrown:
|
||||
|
||||
```sql
|
||||
INSERT INTO test VALUES (42), (43), ('str_1');
|
||||
SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
|
||||
```text
|
||||
┌─d─────┬─dynamicType(d)─┐
|
||||
│ 42 │ Int64 │
|
||||
│ 43 │ Int64 │
|
||||
│ str_1 │ String │
|
||||
└───────┴────────────────┘
|
||||
┌─d─────┬─dynamicType(d)─┐
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
│ [1,2] │ Array(Int64) │
|
||||
│ [3,4] │ Array(Int64) │
|
||||
└───────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
Received exception:
|
||||
Code: 43. DB::Exception: Illegal types Array(Int64) and UInt8 of arguments of function plus: while executing 'FUNCTION plus(__table1.d : 3, 1_UInt8 :: 1) -> plus(__table1.d, 1_UInt8) Dynamic : 0'. (ILLEGAL_TYPE_OF_ARGUMENT)
|
||||
```
|
||||
|
||||
We can filter out unneeded types:
|
||||
|
||||
```sql
|
||||
SELECT d, d + 1 AS res, toTypeName(res), dynamicType(res) FROM test WHERE dynamicType(d) NOT IN ('String', 'Array(Int64)', 'None')
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d──┬─res─┬─toTypeName(res)─┬─dynamicType(res)─┐
|
||||
│ 42 │ 43 │ Dynamic │ Int64 │
|
||||
│ 43 │ 44 │ Dynamic │ Int64 │
|
||||
└────┴─────┴─────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
Or extract required type as subcolumn:
|
||||
|
||||
```sql
|
||||
SELECT d, d.Int64 + 1 AS res, toTypeName(res) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d─────┬──res─┬─toTypeName(res)─┐
|
||||
│ 42 │ 43 │ Nullable(Int64) │
|
||||
│ 43 │ 44 │ Nullable(Int64) │
|
||||
│ str_1 │ ᴺᵁᴸᴸ │ Nullable(Int64) │
|
||||
└───────┴──────┴─────────────────┘
|
||||
┌─d─────┬──res─┬─toTypeName(res)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Nullable(Int64) │
|
||||
│ [1,2] │ ᴺᵁᴸᴸ │ Nullable(Int64) │
|
||||
│ [3,4] │ ᴺᵁᴸᴸ │ Nullable(Int64) │
|
||||
└───────┴──────┴─────────────────┘
|
||||
```
|
||||
|
||||
## Using Dynamic type in ORDER BY and GROUP BY
|
||||
|
||||
During `ORDER BY` and `GROUP BY` values of `Dynamic` types are compared similar to values of `Variant` type:
|
||||
The result of operator `<` for values `d1` with underlying type `T1` and `d2` with underlying type `T2` of a type `Dynamic` is defined as follows:
|
||||
- If `T1 = T2 = T`, the result will be `d1.T < d2.T` (underlying values will be compared).
|
||||
- If `T1 != T2`, the result will be `T1 < T2` (type names will be compared).
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
CREATE TABLE test (d1 Dynamic, d2 Dynamic) ENGINE=Memory;
|
||||
INSERT INTO test VALUES (42, 42), (42, 43), (42, 'abc'), (42, [1, 2, 3]), (42, []), (42, NULL);
|
||||
CREATE TABLE test (d Dynamic) ENGINE=Memory;
|
||||
INSERT INTO test VALUES (42), (43), ('abc'), ('abd'), ([1, 2, 3]), ([]), (NULL);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d2, dynamicType(d2) as d2_type from test order by d2;
|
||||
SELECT d, dynamicType(d) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d2──────┬─d2_type──────┐
|
||||
│ [] │ Array(Int64) │
|
||||
│ [1,2,3] │ Array(Int64) │
|
||||
│ 42 │ Int64 │
|
||||
│ 43 │ Int64 │
|
||||
│ abc │ String │
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
└─────────┴──────────────┘
|
||||
┌─d───────┬─dynamicType(d)─┐
|
||||
│ 42 │ Int64 │
|
||||
│ 43 │ Int64 │
|
||||
│ abc │ String │
|
||||
│ abd │ String │
|
||||
│ [1,2,3] │ Array(Int64) │
|
||||
│ [] │ Array(Int64) │
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
└─────────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d1, dynamicType(d1) as d1_type, d2, dynamicType(d2) as d2_type, d1 = d2, d1 < d2, d1 > d2 from test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d1─┬─d1_type─┬─d2──────┬─d2_type──────┬─equals(d1, d2)─┬─less(d1, d2)─┬─greater(d1, d2)─┐
|
||||
│ 42 │ Int64 │ 42 │ Int64 │ 1 │ 0 │ 0 │
|
||||
│ 42 │ Int64 │ 43 │ Int64 │ 0 │ 1 │ 0 │
|
||||
│ 42 │ Int64 │ abc │ String │ 0 │ 1 │ 0 │
|
||||
│ 42 │ Int64 │ [1,2,3] │ Array(Int64) │ 0 │ 0 │ 1 │
|
||||
│ 42 │ Int64 │ [] │ Array(Int64) │ 0 │ 0 │ 1 │
|
||||
│ 42 │ Int64 │ ᴺᵁᴸᴸ │ None │ 0 │ 1 │ 0 │
|
||||
└────┴─────────┴─────────┴──────────────┴────────────────┴──────────────┴─────────────────┘
|
||||
```
|
||||
|
||||
If you need to find the row with specific `Dynamic` value, you can do one of the following:
|
||||
|
||||
- Cast value to the `Dynamic` type:
|
||||
|
||||
```sql
|
||||
SELECT * FROM test WHERE d2 == [1,2,3]::Array(UInt32)::Dynamic;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d1─┬─d2──────┐
|
||||
│ 42 │ [1,2,3] │
|
||||
└────┴─────────┘
|
||||
```
|
||||
|
||||
- Compare `Dynamic` subcolumn with required type:
|
||||
|
||||
```sql
|
||||
SELECT * FROM test WHERE d2.`Array(Int65)` == [1,2,3] -- or using variantElement(d2, 'Array(UInt32)')
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d1─┬─d2──────┐
|
||||
│ 42 │ [1,2,3] │
|
||||
└────┴─────────┘
|
||||
```
|
||||
|
||||
Sometimes it can be useful to make additional check on dynamic type as subcolumns with complex types like `Array/Map/Tuple` cannot be inside `Nullable` and will have default values instead of `NULL` on rows with different types:
|
||||
|
||||
```sql
|
||||
SELECT d2, d2.`Array(Int64)`, dynamicType(d2) FROM test WHERE d2.`Array(Int64)` == [];
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d2───┬─d2.Array(UInt32)─┬─dynamicType(d2)─┐
|
||||
│ 42 │ [] │ Int64 │
|
||||
│ 43 │ [] │ Int64 │
|
||||
│ abc │ [] │ String │
|
||||
│ [] │ [] │ Array(Int32) │
|
||||
│ ᴺᵁᴸᴸ │ [] │ None │
|
||||
└──────┴──────────────────┴─────────────────┘
|
||||
SELECT d, dynamicType(d) FROM test ORDER BY d;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d2, d2.`Array(Int64)`, dynamicType(d2) FROM test WHERE dynamicType(d2) == 'Array(Int64)' AND d2.`Array(Int64)` == [];
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d2─┬─d2.Array(UInt32)─┬─dynamicType(d2)─┐
|
||||
│ [] │ [] │ Array(Int64) │
|
||||
└────┴──────────────────┴─────────────────┘
|
||||
┌─d───────┬─dynamicType(d)─┐
|
||||
│ [] │ Array(Int64) │
|
||||
│ [1,2,3] │ Array(Int64) │
|
||||
│ 42 │ Int64 │
|
||||
│ 43 │ Int64 │
|
||||
│ abc │ String │
|
||||
│ abd │ String │
|
||||
│ ᴺᵁᴸᴸ │ None │
|
||||
└─────────┴────────────────┘
|
||||
```
|
||||
|
||||
**Note:** values of dynamic types with different numeric types are considered as different values and not compared between each other, their type names are compared instead.
|
||||
@ -411,6 +569,21 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
|
||||
└─────┴────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT d, dynamicType(d) FROM test GROUP by d;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─d───┬─dynamicType(d)─┐
|
||||
│ 1 │ Int64 │
|
||||
│ 100 │ UInt32 │
|
||||
│ 1 │ UInt32 │
|
||||
│ 100 │ Int64 │
|
||||
└─────┴────────────────┘
|
||||
```
|
||||
|
||||
**Note**: the described comparison rule is not applied during execution of comparison functions like `<`/`>`/`=` and others because of [special work](#using-dynamic-type-in-functions) of functions with `Dynamic` type
|
||||
|
||||
## Reaching the limit in number of different data types stored inside Dynamic
|
||||
|
||||
`Dynamic` data type can store only limited number of different data types as separate subcolumns. By default, this limit is 32, but you can change it in type declaration using syntax `Dynamic(max_types=N)` where N is between 0 and 254 (due to implementation details, it's impossible to have more than 254 different data types that can be stored as separate subcolumns inside Dynamic).
|
||||
|
||||
@ -226,9 +226,9 @@ Result:
|
||||
|
||||
## bitTestAll
|
||||
|
||||
Returns result of [logical conjuction](https://en.wikipedia.org/wiki/Logical_conjunction) (AND operator) of all bits at given positions. Counting is right-to-left, starting at 0.
|
||||
Returns result of [logical conjunction](https://en.wikipedia.org/wiki/Logical_conjunction) (AND operator) of all bits at given positions. Counting is right-to-left, starting at 0.
|
||||
|
||||
The conjuction for bit-wise operations:
|
||||
The conjunction for bit-wise operations:
|
||||
|
||||
0 AND 0 = 0
|
||||
|
||||
@ -251,7 +251,7 @@ SELECT bitTestAll(number, index1, index2, index3, index4, ...)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Result of the logical conjuction. [UInt8](../data-types/int-uint.md).
|
||||
- Result of the logical conjunction. [UInt8](../data-types/int-uint.md).
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -1623,12 +1623,22 @@ If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Mo
|
||||
toStartOfInterval(value, INTERVAL x unit[, time_zone])
|
||||
toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]])
|
||||
```
|
||||
Aliases: `time_bucket`, `date_bin`.
|
||||
|
||||
The second overload emulates TimescaleDB's `time_bucket()` function, respectively PostgreSQL's `date_bin()` function, e.g.
|
||||
|
||||
``` SQL
|
||||
SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` reference
|
||||
┌───toStartOfInterval(...)─┐
|
||||
│ 2023-01-01 14:44:30 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
- [date_trunc](#date_trunc)
|
||||
|
||||
@ -1962,7 +1972,39 @@ Result:
|
||||
|
||||
## toISOYear
|
||||
|
||||
Converts a date, or date with time, to a UInt16 number containing the ISO Year number.
|
||||
Converts a date, or date with time, to the ISO year as a UInt16 number.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toISOYear(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — The value with date or date with time. [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The input value converted to a ISO year number. [UInt16](../data-types/int-uint.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toISOYear(toDate('2024/10/02')) as year1,
|
||||
toISOYear(toDateTime('2024-10-02 01:30:00')) as year2
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─year1─┬─year2─┐
|
||||
│ 2024 │ 2024 │
|
||||
└───────┴───────┘
|
||||
```
|
||||
|
||||
## toISOWeek
|
||||
|
||||
|
||||
@ -51,6 +51,40 @@ Calculates the MD5 from a string and returns the resulting set of bytes as Fixed
|
||||
If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead.
|
||||
If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))).
|
||||
|
||||
## RIPEMD160
|
||||
|
||||
Produces [RIPEMD-160](https://en.wikipedia.org/wiki/RIPEMD) hash value.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
RIPEMD160(input)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `input`: Input string. [String](../data-types/string.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A 160-bit `RIPEMD-160` hash value of type [FixedString(20)](../data-types/fixedstring.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Use the [hex](../functions/encoding-functions.md/#hex) function to represent the result as a hex-encoded string.
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'));
|
||||
```
|
||||
|
||||
```response
|
||||
┌─HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐
|
||||
│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## sipHash64
|
||||
|
||||
Produces a 64-bit [SipHash](https://en.wikipedia.org/wiki/SipHash) hash value.
|
||||
|
||||
@ -280,10 +280,74 @@ SELECT
|
||||
|
||||
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toIPv4OrDefault(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toIPv4OrDefault('192.168.0.1') AS s1,
|
||||
toIPv4OrDefault('192.168.0') AS s2
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─s1──────────┬─s2──────┐
|
||||
│ 192.168.0.1 │ 0.0.0.0 │
|
||||
└─────────────┴─────────┘
|
||||
```
|
||||
|
||||
## toIPv4OrNull(string)
|
||||
|
||||
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toIPv4OrNull(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toIPv4OrNull('192.168.0.1') AS s1,
|
||||
toIPv4OrNull('192.168.0') AS s2
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─s1──────────┬─s2───┐
|
||||
│ 192.168.0.1 │ ᴺᵁᴸᴸ │
|
||||
└─────────────┴──────┘
|
||||
```
|
||||
|
||||
## toIPv6OrDefault(string)
|
||||
|
||||
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns `::` (0 IPv6).
|
||||
|
||||
@ -202,12 +202,36 @@ Result:
|
||||
|
||||
Returns the type name of the passed argument.
|
||||
|
||||
If `NULL` is passed, then the function returns type `Nullable(Nothing)`, which corresponds to ClickHouse's internal `NULL` representation.
|
||||
If `NULL` is passed, the function returns type `Nullable(Nothing)`, which corresponds to ClickHouse's internal `NULL` representation.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toTypeName(x)
|
||||
toTypeName(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — A value of arbitrary type.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The data type name of the input value. [String](../data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toTypeName(123);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toTypeName(123)─┐
|
||||
│ UInt8 │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## blockSize {#blockSize}
|
||||
@ -386,13 +410,37 @@ Code: 44. DB::Exception: Received from localhost:9000. DB::Exception: Illegal ty
|
||||
|
||||
## ignore
|
||||
|
||||
Accepts any arguments, including `NULL` and does nothing. Always returns 0.
|
||||
The argument is internally still evaluated. Useful e.g. for benchmarks.
|
||||
Accepts arbitrary arguments and unconditionally returns `0`.
|
||||
The argument is still evaluated internally, making it useful for eg. benchmarking.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
ignore(x)
|
||||
ignore([arg1[, arg2[, ...]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- Accepts arbitrarily many arguments of arbitrary type, including `NULL`.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns `0`.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT ignore(0, 'ClickHouse', NULL);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─ignore(0, 'ClickHouse', NULL)─┐
|
||||
│ 0 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
## sleep
|
||||
@ -500,6 +548,26 @@ Useful in table engine parameters of `CREATE TABLE` queries where you need to sp
|
||||
currentDatabase()
|
||||
```
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns the current database name. [String](../data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT currentDatabase()
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─currentDatabase()─┐
|
||||
│ default │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## currentUser {#currentUser}
|
||||
|
||||
Returns the name of the current user. In case of a distributed query, the name of the user who initiated the query is returned.
|
||||
@ -531,6 +599,42 @@ Result:
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## currentSchemas
|
||||
|
||||
Returns a single-element array with the name of the current database schema.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
currentSchemas(bool)
|
||||
```
|
||||
|
||||
Alias: `current_schemas`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `bool`: A boolean value. [Bool](../data-types/boolean.md).
|
||||
|
||||
:::note
|
||||
The boolean argument is ignored. It only exists for the sake of compatibility with the [implementation](https://www.postgresql.org/docs/7.3/functions-misc.html) of this function in PostgreSQL.
|
||||
:::
|
||||
|
||||
**Returned values**
|
||||
|
||||
- Returns a single-element array with the name of the current database
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT currentSchemas(true);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
['default']
|
||||
```
|
||||
|
||||
## isConstant
|
||||
|
||||
Returns whether the argument is a constant expression.
|
||||
@ -1773,7 +1877,7 @@ toColumnTypeName(value)
|
||||
|
||||
**Example**
|
||||
|
||||
Difference between `toTypeName ' and ' toColumnTypeName`:
|
||||
Difference between `toTypeName` and `toColumnTypeName`:
|
||||
|
||||
```sql
|
||||
SELECT toTypeName(CAST('2018-01-01 01:02:03' AS DateTime))
|
||||
@ -2789,6 +2893,45 @@ Result:
|
||||
|
||||
- [Custom Settings](../../operations/settings/index.md#custom_settings)
|
||||
|
||||
## getSettingOrDefault
|
||||
|
||||
Returns the current value of a [custom setting](../../operations/settings/index.md#custom_settings) or returns the default value specified in the 2nd argument if the custom setting is not set in the current profile.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
getSettingOrDefault('custom_setting', default_value);
|
||||
```
|
||||
|
||||
**Parameter**
|
||||
|
||||
- `custom_setting` — The setting name. [String](../data-types/string.md).
|
||||
- `default_value` — Value to return if custom_setting is not set. Value may be of any data type or Null.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The setting's current value or default_value if setting is not set.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT getSettingOrDefault('custom_undef1', 'my_value');
|
||||
SELECT getSettingOrDefault('custom_undef2', 100);
|
||||
SELECT getSettingOrDefault('custom_undef3', NULL);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
my_value
|
||||
100
|
||||
NULL
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Custom Settings](../../operations/settings/index.md#custom_settings)
|
||||
|
||||
## isDecimalOverflow
|
||||
|
||||
Checks whether the [Decimal](../data-types/decimal.md) value is outside its precision or outside the specified precision.
|
||||
@ -3810,13 +3953,15 @@ Retrieves the connection ID of the client that submitted the current query and r
|
||||
connectionId()
|
||||
```
|
||||
|
||||
Alias: `connection_id`.
|
||||
|
||||
**Parameters**
|
||||
|
||||
None.
|
||||
|
||||
**Returned value**
|
||||
|
||||
Returns an integer of type UInt64.
|
||||
The current connection ID. [UInt64](../data-types/int-uint.md).
|
||||
|
||||
**Implementation details**
|
||||
|
||||
@ -3834,40 +3979,6 @@ SELECT connectionId();
|
||||
0
|
||||
```
|
||||
|
||||
## connection_id
|
||||
|
||||
An alias of `connectionId`. Retrieves the connection ID of the client that submitted the current query and returns it as a UInt64 integer.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
connection_id()
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
None.
|
||||
|
||||
**Returned value**
|
||||
|
||||
Returns an integer of type UInt64.
|
||||
|
||||
**Implementation details**
|
||||
|
||||
This function is most useful in debugging scenarios or for internal purposes within the MySQL handler. It was created for compatibility with [MySQL's `CONNECTION_ID` function](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_connection-id) It is not typically used in production queries.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT connection_id();
|
||||
```
|
||||
|
||||
```response
|
||||
0
|
||||
```
|
||||
|
||||
## getClientHTTPHeader
|
||||
|
||||
Get the value of an HTTP header.
|
||||
|
||||
@ -288,7 +288,7 @@ roundToExp2(num)
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `num`: A number representing an age in years. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
|
||||
- `num`: A number to round. [UInt](../data-types/int-uint.md)/[Float](../data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -755,7 +755,7 @@ Result:
|
||||
|
||||
## match {#match}
|
||||
|
||||
Returns whether string `haystack` matches the regular expression `pattern` in [re2 regular syntax](https://github.com/google/re2/wiki/Syntax).
|
||||
Returns whether string `haystack` matches the regular expression `pattern` in [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
|
||||
|
||||
Matching is based on UTF-8, e.g. `.` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. The regular
|
||||
expression must not contain null bytes. If the haystack or the pattern are not valid UTF-8, then the behavior is undefined.
|
||||
@ -852,9 +852,10 @@ multiFuzzyMatchAllIndices(haystack, distance, \[pattern<sub>1</sub>, pattern<sub
|
||||
|
||||
## extract
|
||||
|
||||
Extracts a fragment of a string using a regular expression. If `haystack` does not match the `pattern` regex, an empty string is returned.
|
||||
Returns the first match of a regular expression in a string.
|
||||
If `haystack` does not match the `pattern` regex, an empty string is returned.
|
||||
|
||||
For regex without subpatterns, the function uses the fragment that matches the entire regex. Otherwise, it uses the fragment that matches the first subpattern.
|
||||
If the regular expression has capturing groups, the function matches the input string against the first capturing group.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -862,13 +863,36 @@ For regex without subpatterns, the function uses the fragment that matches the e
|
||||
extract(haystack, pattern)
|
||||
```
|
||||
|
||||
*Arguments**
|
||||
|
||||
- `haystack` — Input string. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The first match of the regular expression in the haystack string. [String](../data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT extract('number: 1, number: 2, number: 3', '\\d+') AS result;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─result─┐
|
||||
│ 1 │
|
||||
└────────┘
|
||||
```
|
||||
|
||||
## extractAll
|
||||
|
||||
Extracts all fragments of a string using a regular expression. If `haystack` does not match the `pattern` regex, an empty string is returned.
|
||||
Returns an array of all matches of a regular expression in a string. If `haystack` does not match the `pattern` regex, an empty string is returned.
|
||||
|
||||
Returns an array of strings consisting of all matches of the regex.
|
||||
|
||||
The behavior with respect to subpatterns is the same as in function `extract`.
|
||||
The behavior with respect to sub-patterns is the same as in function [`extract`](#extract).
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -876,6 +900,31 @@ The behavior with respect to subpatterns is the same as in function `extract`.
|
||||
extractAll(haystack, pattern)
|
||||
```
|
||||
|
||||
*Arguments**
|
||||
|
||||
- `haystack` — Input string. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Array of matches of the regular expression in the haystack string. [Array](../data-types/array.md)([String](../data-types/string.md)).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT extractAll('number: 1, number: 2, number: 3', '\\d+') AS result;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─result────────┐
|
||||
│ ['1','2','3'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## extractAllGroupsHorizontal
|
||||
|
||||
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where the first array includes all fragments matching the first group, the second array - matching the second group, etc.
|
||||
@ -891,7 +940,7 @@ extractAllGroupsHorizontal(haystack, pattern)
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — Input string. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -915,6 +964,39 @@ Result:
|
||||
└──────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## extractGroups
|
||||
|
||||
Match all groups of given input string with a given regular expression, returns an array of arrays of matches.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
extractGroups(haystack, pattern)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — Input string. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Array of arrays of matches. [Array](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT extractGroups('hello abc=111 world', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') AS result;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─result────────┐
|
||||
│ ['abc','111'] │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## extractAllGroupsVertical
|
||||
|
||||
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where each array includes matching fragments from every group. Fragments are grouped in order of appearance in the `haystack`.
|
||||
@ -928,7 +1010,7 @@ extractAllGroupsVertical(haystack, pattern)
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — Input string. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
|
||||
- `pattern` — Regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). Must contain groups, each group enclosed in parentheses. If `pattern` contains no groups, an exception is thrown. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -1484,7 +1566,7 @@ countMatches(haystack, pattern)
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
|
||||
- `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -1529,7 +1611,7 @@ countMatchesCaseInsensitive(haystack, pattern)
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
|
||||
- `pattern` — The regular expression with [re2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -4137,6 +4137,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal32(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal32('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(9, S)`. [Decimal32(S)](../data-types/int-uint.md).
|
||||
@ -4328,6 +4334,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal32OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal32OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(9, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal32(S)](../data-types/decimal.md).
|
||||
@ -4391,6 +4403,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal64(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal64('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(18, S)`. [Decimal64(S)](../data-types/int-uint.md).
|
||||
@ -4582,6 +4600,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal64OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal64OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(18, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal64(S)](../data-types/decimal.md).
|
||||
@ -4645,6 +4669,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal128(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal128('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(38, S)`. [Decimal128(S)](../data-types/int-uint.md).
|
||||
@ -4836,6 +4866,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal128OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal128OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(38, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal128(S)](../data-types/decimal.md).
|
||||
@ -4899,6 +4935,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an exception.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal256(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal256('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(76, S)`. [Decimal256(S)](../data-types/int-uint.md).
|
||||
@ -5090,6 +5132,12 @@ Excessive digits in a fraction are discarded (not rounded).
|
||||
Excessive digits in the integer part will lead to an error.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
Conversions drop extra digits and could operate in an unexpected way when working with Float32/Float64 inputs as the operations are performed using floating point instructions.
|
||||
For example: `toDecimal256OrDefault(1.15, 2)` is equal to `1.14` because 1.15 * 100 in floating point is 114.99.
|
||||
You can use a String input so the operations use the underlying integer type: `toDecimal256OrDefault('1.15', 2) = 1.15`
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Value of type `Decimal(76, S)` if successful, otherwise returns the default value if passed or `0` if not. [Decimal256(S)](../data-types/decimal.md).
|
||||
|
||||
@ -272,8 +272,7 @@ ALTER TABLE table_name MODIFY COLUMN column_name RESET SETTING max_compress_bloc
|
||||
|
||||
## MATERIALIZE COLUMN
|
||||
|
||||
Materializes a column with a `DEFAULT` or `MATERIALIZED` value expression.
|
||||
This statement can be used to rewrite existing column data after a `DEFAULT` or `MATERIALIZED` expression has been added or updated (which only updates the metadata but does not change existing data).
|
||||
Materializes a column with a `DEFAULT` or `MATERIALIZED` value expression. When adding a materialized column using `ALTER TABLE table_name ADD COLUMN column_name MATERIALIZED`, existing rows without materialized values are not automatically filled. `MATERIALIZE COLUMN` statement can be used to rewrite existing column data after a `DEFAULT` or `MATERIALIZED` expression has been added or updated (which only updates the metadata but does not change existing data).
|
||||
Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
For columns with a new or updated `MATERIALIZED` value expression, all existing rows are rewritten.
|
||||
|
||||
@ -12,10 +12,10 @@ Syntax:
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]]
|
||||
[ON CLUSTER cluster_name]
|
||||
[NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}]
|
||||
[NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}
|
||||
[, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
|
||||
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[RESET AUTHENTICATION METHODS TO NEW]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
|
||||
@ -43,6 +43,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine]
|
||||
|
||||
Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table.
|
||||
|
||||
### With a Schema and Data Cloned from Another Table
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 [ENGINE = engine]
|
||||
```
|
||||
|
||||
Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine];
|
||||
ALTER TABLE [db.]table_name ATTACH PARTITION ALL FROM [db2].name2;
|
||||
```
|
||||
|
||||
### From a Table Function
|
||||
|
||||
``` sql
|
||||
|
||||
@ -11,10 +11,10 @@ Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}
|
||||
[, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
|
||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[RESET AUTHENTICATION METHODS TO NEW]
|
||||
[IN access_storage_type]
|
||||
[DEFAULT ROLE role [,...]]
|
||||
[DEFAULT DATABASE database | NONE]
|
||||
|
||||
@ -135,15 +135,15 @@ To change SQL security for an existing view, use
|
||||
ALTER TABLE MODIFY SQL SECURITY { DEFINER | INVOKER | NONE } [DEFINER = { user | CURRENT_USER }]
|
||||
```
|
||||
|
||||
### Examples sql security
|
||||
### Examples
|
||||
```sql
|
||||
CREATE test_view
|
||||
CREATE VIEW test_view
|
||||
DEFINER = alice SQL SECURITY DEFINER
|
||||
AS SELECT ...
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE test_view
|
||||
CREATE VIEW test_view
|
||||
SQL SECURITY INVOKER
|
||||
AS SELECT ...
|
||||
```
|
||||
@ -184,14 +184,6 @@ Differences from regular non-refreshable materialized views:
|
||||
The settings in the `REFRESH ... SETTINGS` part of the query are refresh settings (e.g. `refresh_retries`), distinct from regular settings (e.g. `max_threads`). Regular settings can be specified using `SETTINGS` at the end of the query.
|
||||
:::
|
||||
|
||||
:::note
|
||||
Refreshable materialized views are a work in progress. Setting `allow_experimental_refreshable_materialized_view = 1` is required for creating one. Current limitations:
|
||||
* not compatible with Replicated database or table engines
|
||||
* It is not supported in ClickHouse Cloud
|
||||
* require [Atomic database engine](../../../engines/database-engines/atomic.md),
|
||||
* no limit on number of concurrent refreshes.
|
||||
:::
|
||||
|
||||
### Refresh Schedule
|
||||
|
||||
Example refresh schedules:
|
||||
@ -203,6 +195,10 @@ REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at
|
||||
REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc
|
||||
REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day
|
||||
-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax error, OFFSET is not allowed with AFTER
|
||||
REFRESH EVERY 1 WEEK 2 DAYS -- every 9 days, not on any particular day of the week or month;
|
||||
-- specifically, when day number (since 1969-12-29) is divisible by 9
|
||||
REFRESH EVERY 5 MONTHS -- every 5 months, different months each year (as 12 is not divisible by 5);
|
||||
-- specifically, when month number (since 1970-01) is divisible by 5
|
||||
```
|
||||
|
||||
`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.:
|
||||
@ -214,6 +210,16 @@ At most one refresh may be running at a time, for a given view. E.g. if a view w
|
||||
|
||||
Additionally, a refresh is started immediately after the materialized view is created, unless `EMPTY` is specified in the `CREATE` query. If `EMPTY` is specified, the first refresh happens according to schedule.
|
||||
|
||||
### In Replicated DB
|
||||
|
||||
If the refreshable materialized view is in a [Replicated database](../../../engines/database-engines/replicated.md), the replicas coordinate with each other such that only one replica performs the refresh at each scheduled time. [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md) table engine is required, so that all replicas see the data produced by the refresh.
|
||||
|
||||
In `APPEND` mode, coordination can be disabled using `SETTINGS all_replicas = 1`. This makes replicas do refreshes independently of each other. In this case ReplicatedMergeTree is not required.
|
||||
|
||||
In non-`APPEND` mode, only coordinated refreshing is supported. For uncoordinated, use `Atomic` database and `CREATE ... ON CLUSTER` query to create refreshable materialized views on all replicas.
|
||||
|
||||
The coordination is done through Keeper. The znode path is determined by [default_replica_path](../../../operations/server-configuration-parameters/settings.md#default_replica_path) server setting.
|
||||
|
||||
### Dependencies {#refresh-dependencies}
|
||||
|
||||
`DEPENDS ON` synchronizes refreshes of different tables. By way of example, suppose there's a chain of two refreshable materialized views:
|
||||
@ -277,6 +283,8 @@ The status of all refreshable materialized views is available in table [`system.
|
||||
|
||||
To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|CANCEL VIEW`](../system.md#refreshable-materialized-views).
|
||||
|
||||
To wait for a refresh to complete, use [`SYSTEM WAIT VIEW`](../system.md#refreshable-materialized-views). In particular, useful for waiting for initial refresh after creating a view.
|
||||
|
||||
:::note
|
||||
Fun fact: the refresh query is allowed to read from the view that's being refreshed, seeing pre-refresh version of the data. This means you can implement Conway's game of life: https://pastila.nl/?00021a4b/d6156ff819c83d490ad2dcec05676865#O0LGWTO7maUQIA4AcGUtlA==
|
||||
:::
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: EXISTS
|
||||
# EXISTS Statement
|
||||
|
||||
``` sql
|
||||
EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format]
|
||||
EXISTS [TEMPORARY] [TABLE|DICTIONARY|DATABASE] [db.]name [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
Returns a single `UInt8`-type column, which contains the single value `0` if the table or database does not exist, or `1` if the table exists in the specified database.
|
||||
|
||||
@ -233,15 +233,20 @@ Hierarchy of privileges:
|
||||
- `addressToSymbol`
|
||||
- `demangle`
|
||||
- [SOURCES](#sources)
|
||||
- `AZURE`
|
||||
- `FILE`
|
||||
- `URL`
|
||||
- `REMOTE`
|
||||
- `YSQL`
|
||||
- `ODBC`
|
||||
- `JDBC`
|
||||
- `HDFS`
|
||||
- `S3`
|
||||
- `HIVE`
|
||||
- `JDBC`
|
||||
- `MONGO`
|
||||
- `MYSQL`
|
||||
- `ODBC`
|
||||
- `POSTGRES`
|
||||
- `REDIS`
|
||||
- `REMOTE`
|
||||
- `S3`
|
||||
- `SQLITE`
|
||||
- `URL`
|
||||
- [dictGet](#dictget)
|
||||
- [displaySecretsInShowAndSelect](#displaysecretsinshowandselect)
|
||||
- [NAMED COLLECTION ADMIN](#named-collection-admin)
|
||||
@ -510,15 +515,20 @@ Allows using [introspection](../../operations/optimizing-performance/sampling-qu
|
||||
Allows using external data sources. Applies to [table engines](../../engines/table-engines/index.md) and [table functions](../../sql-reference/table-functions/index.md#table-functions).
|
||||
|
||||
- `SOURCES`. Level: `GROUP`
|
||||
- `AZURE`. Level: `GLOBAL`
|
||||
- `FILE`. Level: `GLOBAL`
|
||||
- `URL`. Level: `GLOBAL`
|
||||
- `REMOTE`. Level: `GLOBAL`
|
||||
- `YSQL`. Level: `GLOBAL`
|
||||
- `ODBC`. Level: `GLOBAL`
|
||||
- `JDBC`. Level: `GLOBAL`
|
||||
- `HDFS`. Level: `GLOBAL`
|
||||
- `S3`. Level: `GLOBAL`
|
||||
- `HIVE`. Level: `GLOBAL`
|
||||
- `JDBC`. Level: `GLOBAL`
|
||||
- `MONGO`. Level: `GLOBAL`
|
||||
- `MYSQL`. Level: `GLOBAL`
|
||||
- `ODBC`. Level: `GLOBAL`
|
||||
- `POSTGRES`. Level: `GLOBAL`
|
||||
- `REDIS`. Level: `GLOBAL`
|
||||
- `REMOTE`. Level: `GLOBAL`
|
||||
- `S3`. Level: `GLOBAL`
|
||||
- `SQLITE`. Level: `GLOBAL`
|
||||
- `URL`. Level: `GLOBAL`
|
||||
|
||||
The `SOURCES` privilege enables use of all the sources. Also you can grant a privilege for each source individually. To use sources, you need additional privileges.
|
||||
|
||||
|
||||
@ -58,7 +58,9 @@ KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
|
||||
KILL QUERY WHERE user='username' SYNC
|
||||
```
|
||||
|
||||
:::tip If you are killing a query in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the query is killed on all replicas:::
|
||||
:::tip
|
||||
If you are killing a query in ClickHouse Cloud or in a self-managed cluster, then be sure to use the ```ON CLUSTER [cluster-name]``` option, in order to ensure the query is killed on all replicas
|
||||
:::
|
||||
|
||||
Read-only users can only stop their own queries.
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ The condition could be any expression based on your requirements.
|
||||
Here is a simple example that intersects the numbers 1 to 10 with the numbers 3 to 8:
|
||||
|
||||
```sql
|
||||
SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,6);
|
||||
SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,8);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
@ -351,11 +351,15 @@ Shows privileges for a user.
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
SHOW GRANTS [FOR user1 [, user2 ...]]
|
||||
SHOW GRANTS [FOR user1 [, user2 ...]] [WITH IMPLICIT] [FINAL]
|
||||
```
|
||||
|
||||
If user is not specified, the query returns privileges for the current user.
|
||||
|
||||
The `WITH IMPLICIT` modifier allows to show the implicit grants (e.g., `GRANT SELECT ON system.one`)
|
||||
|
||||
The `FINAL` modifier merges all grants from the user and its granted roles (with inheritance)
|
||||
|
||||
## SHOW CREATE USER
|
||||
|
||||
Shows parameters that were used at a [user creation](../../sql-reference/statements/create/user.md).
|
||||
|
||||
@ -565,3 +565,13 @@ If there's a refresh in progress for the given view, interrupt and cancel it. Ot
|
||||
```sql
|
||||
SYSTEM CANCEL VIEW [db.]name
|
||||
```
|
||||
|
||||
### SYSTEM WAIT VIEW
|
||||
|
||||
Waits for the running refresh to complete. If no refresh is running, returns immediately. If the latest refresh attempt failed, reports an error.
|
||||
|
||||
Can be used right after creating a new refreshable materialized view (without EMPTY keyword) to wait for the initial refresh to complete.
|
||||
|
||||
```sql
|
||||
SYSTEM WAIT VIEW [db.]name
|
||||
```
|
||||
|
||||
@ -6,7 +6,7 @@ sidebar_label: "Настройки пользователей"
|
||||
|
||||
# Настройки пользователей {#nastroiki-polzovatelei}
|
||||
|
||||
Раздел `users` конфигурационного файла `user.xml` содержит настройки для пользователей.
|
||||
Раздел `users` конфигурационного файла `users.xml` содержит настройки для пользователей.
|
||||
|
||||
:::note Информация
|
||||
Для управления пользователями рекомендуется использовать [SQL-ориентированный воркфлоу](../access-rights.md#access-control), который также поддерживается в ClickHouse.
|
||||
|
||||
@ -468,6 +468,52 @@ WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 SELECT toStartOfSecond(d
|
||||
`toStartOfInterval(t, INTERVAL 1 day)` возвращает то же самое, что и `toStartOfDay(t)`,
|
||||
`toStartOfInterval(t, INTERVAL 15 minute)` возвращает то же самое, что и `toStartOfFifteenMinutes(t)`, и т.п.
|
||||
|
||||
Вычисление происхожит относительно конкретного времени:
|
||||
|
||||
| Interval | Start |
|
||||
|-------------|------------------------|
|
||||
| YEAR | year 0 |
|
||||
| QUARTER | 1900 Q1 |
|
||||
| MONTH | 1900 January |
|
||||
| WEEK | 1970, 1st week (01-05) |
|
||||
| DAY | 1970-01-01 |
|
||||
| HOUR | (*) |
|
||||
| MINUTE | 1970-01-01 00:00:00 |
|
||||
| SECOND | 1970-01-01 00:00:00 |
|
||||
| MILLISECOND | 1970-01-01 00:00:00 |
|
||||
| MICROSECOND | 1970-01-01 00:00:00 |
|
||||
| NANOSECOND | 1970-01-01 00:00:00 |
|
||||
|
||||
(*) часовые интервалы особенные: Вычисления всегда происходят относительно 00:00:00 (полночь) текущего дня. В результате, только
|
||||
часовые значения полезны только между 1 и 23.
|
||||
|
||||
Если была указана единица измерения `WEEK`, то `toStartOfInterval` предполагает, что неделя начинается в понедельник. Важно, что это поведение отдичается от поведения функции `toStartOfWeek` в которой неделя по умолчанию начинается у воскресенье.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
toStartOfInterval(value, INTERVAL x unit[, time_zone])
|
||||
toStartOfInterval(value, INTERVAL x unit[, origin[, time_zone]])
|
||||
```
|
||||
Синонимы: `time_bucket`, `date_bin`.
|
||||
|
||||
Вторая перегрузка эмулирует функцию `time_bucket()` из TimescaleDB, и функцию `date_bin()` из PostgreSQL, например:
|
||||
|
||||
``` SQL
|
||||
SELECT toStartOfInterval(toDateTime('2023-01-01 14:45:00'), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'));
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` reference
|
||||
┌───toStartOfInterval(...)─┐
|
||||
│ 2023-01-01 14:44:30 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
**См. также**
|
||||
- [date_trunc](#date_trunc)
|
||||
|
||||
## toTime {#totime}
|
||||
|
||||
Переводит дату-с-временем на некоторую фиксированную дату, сохраняя при этом время.
|
||||
|
||||
@ -13,8 +13,10 @@ sidebar_label: USER
|
||||
``` sql
|
||||
ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]]
|
||||
[ON CLUSTER cluster_name]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}]
|
||||
[NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}
|
||||
[, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
|
||||
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||
|
||||
@ -12,8 +12,11 @@ sidebar_label: "Пользователь"
|
||||
|
||||
``` sql
|
||||
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}]
|
||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}
|
||||
[, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]]
|
||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||
[VALID UNTIL datetime]
|
||||
[IN access_storage_type]
|
||||
[DEFAULT ROLE role [,...]]
|
||||
[DEFAULT DATABASE database | NONE]
|
||||
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||
|
||||
@ -29,7 +29,7 @@ FROM table2
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,6);
|
||||
SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,8);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user