thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into break_compatibility

Browse Source
This commit is contained in:
Alexander Tokmakov 2024-07-05 16:35:52 +02:00
commit ac17c7194d
129 changed files with 3066 additions and 1445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
base/poco/Foundation/include/Poco/Logger.h
View File

@ -21,6 +21,8 @@
#include <atomic>
#include <cstddef>
#include <map>
#include <memory>
#include <unordered_map>
#include <vector>
#include "Poco/Channel.h"

1
base/poco/Foundation/include/Poco/Message.h
View File

@ -19,6 +19,7 @@
#include <map>
#include <vector>
#include "Poco/Foundation.h"
#include "Poco/Timestamp.h"

2
contrib/CMakeLists.txt vendored
View File

@ -228,6 +228,8 @@ add_contrib (ulid-c-cmake ulid-c)
add_contrib (libssh-cmake libssh)
add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

2
contrib/azure vendored

@ -1 +1 @@
Subproject commit 6262a76ef4c4c330c84e58dd4f6f13f4e6230fcd
Subproject commit 92c94d7f37a43cc8fc4d466884a95f610c0593bf

51
contrib/google-protobuf-cmake/protobuf_generate.cmake
View File

@ -157,15 +157,13 @@ function(protobuf_generate)
set(_generated_srcs_all)
foreach(_proto ${protobuf_generate_PROTOS})
get_filename_component(_abs_file ${_proto} ABSOLUTE)
get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
get_filename_component(_basename ${_proto} NAME_WE)
file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
set(_possible_rel_dir)
if (NOT protobuf_generate_APPEND_PATH)
set(_possible_rel_dir ${_rel_dir}/)
endif()
# The protobuf compiler doesn't return paths to the files it generates so we have to calculate those paths here:
# _abs_file - absolute path to a .proto file,
# _possible_rel_dir - relative path to the .proto file from some import directory specified in Protobuf_IMPORT_DIRS,
# _basename - filename of the .proto file (without path and without extenstion).
get_proto_absolute_path(_abs_file "${_proto}" ${_protobuf_include_path})
get_proto_relative_path(_possible_rel_dir "${_abs_file}" ${_protobuf_include_path})
get_filename_component(_basename "${_abs_file}" NAME_WE)
set(_generated_srcs)
foreach(_ext ${protobuf_generate_GENERATE_EXTENSIONS})
@ -173,7 +171,7 @@ function(protobuf_generate)
endforeach()
if(protobuf_generate_DESCRIPTORS AND protobuf_generate_LANGUAGE STREQUAL cpp)
set(_descriptor_file "${CMAKE_CURRENT_BINARY_DIR}/${_basename}.desc")
set(_descriptor_file "${protobuf_generate_PROTOC_OUT_DIR}/${_possible_rel_dir}${_basename}.desc")
set(_dll_desc_out "--descriptor_set_out=${_descriptor_file}")
list(APPEND _generated_srcs ${_descriptor_file})
endif()
@ -196,3 +194,36 @@ function(protobuf_generate)
target_sources(${protobuf_generate_TARGET} PRIVATE ${_generated_srcs_all})
endif()
endfunction()
# Calculates the absolute path to a .proto file.
function(get_proto_absolute_path result proto)
cmake_path(IS_ABSOLUTE proto _is_abs_path)
if(_is_abs_path)
set(${result} "${proto}" PARENT_SCOPE)
return()
endif()
foreach(_include_dir ${ARGN})
if(EXISTS "${_include_dir}/${proto}")
set(${result} "${_include_dir}/${proto}" PARENT_SCOPE)
return()
endif()
endforeach()
message(SEND_ERROR "Not found protobuf ${proto} in Protobuf_IMPORT_DIRS: ${ARGN}")
endfunction()
# Calculates a relative path to a .proto file. The returned path is relative to one of include directories.
function(get_proto_relative_path result abs_path)
set(${result} "" PARENT_SCOPE)
get_filename_component(_abs_dir "${abs_path}" DIRECTORY)
foreach(_include_dir ${ARGN})
cmake_path(IS_PREFIX _include_dir "${_abs_dir}" _is_prefix)
if(_is_prefix)
file(RELATIVE_PATH _rel_dir "${_include_dir}" "${_abs_dir}")
if(NOT _rel_dir STREQUAL "")
set(${result} "${_rel_dir}/" PARENT_SCOPE)
endif()
return()
endif()
endforeach()
message(WARNING "Not found protobuf ${abs_path} in Protobuf_IMPORT_DIRS: ${ARGN}")
endfunction()

2
contrib/icu-cmake/CMakeLists.txt
View File

@ -5,7 +5,7 @@ else ()
endif ()
if (NOT ENABLE_ICU)
message(STATUS "Not using icu")
message(STATUS "Not using ICU")
return()
endif()

4
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -34,7 +34,11 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
endif()
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
endif()

34
contrib/prometheus-protobufs-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,34 @@
option(ENABLE_PROMETHEUS_PROTOBUFS "Enable Prometheus Protobufs" ${ENABLE_PROTOBUF})
if(NOT ENABLE_PROMETHEUS_PROTOBUFS)
message(STATUS "Not using prometheus-protobufs")
return()
endif()
set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
set(Prometheus_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/prometheus-protobufs")
set(GogoProto_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/prometheus-protobufs-gogo")
# Protobuf_IMPORT_DIRS specify where the protobuf compiler will look for .proto files.
set(Old_Protobuf_IMPORT_DIRS ${Protobuf_IMPORT_DIRS})
list(APPEND Protobuf_IMPORT_DIRS "${Protobuf_INCLUDE_DIR}" "${Prometheus_INCLUDE_DIR}" "${GogoProto_INCLUDE_DIR}")
PROTOBUF_GENERATE_CPP(prometheus_protobufs_sources prometheus_protobufs_headers
"prompb/remote.proto"
"prompb/types.proto"
"gogoproto/gogo.proto"
)
set(Protobuf_IMPORT_DIRS ${Old_Protobuf_IMPORT_DIRS})
# Ignore warnings while compiling protobuf-generated *.pb.h and *.pb.cpp files.
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
# Disable clang-tidy for protobuf-generated *.pb.h and *.pb.cpp files.
set (CMAKE_CXX_CLANG_TIDY "")
add_library(_prometheus_protobufs ${prometheus_protobufs_sources} ${prometheus_protobufs_headers})
target_include_directories(_prometheus_protobufs SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
target_link_libraries (_prometheus_protobufs PUBLIC ch_contrib::protobuf)
add_library (ch_contrib::prometheus_protobufs ALIAS _prometheus_protobufs)

35
contrib/prometheus-protobufs-gogo/LICENSE Normal file
View File

@ -0,0 +1,35 @@
Copyright (c) 2022, The Cosmos SDK Authors. All rights reserved.
Copyright (c) 2013, The GoGo Authors. All rights reserved.
Protocol Buffers for Go with Gadgets
Go support for Protocol Buffers - Google's data interchange format
Copyright 2010 The Go Authors. All rights reserved.
https://github.com/golang/protobuf
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

4
contrib/prometheus-protobufs-gogo/README Normal file
View File

@ -0,0 +1,4 @@
File "gogoproto/gogo.proto" was downloaded from the "Protocol Buffers for Go with Gadgets" project:
https://github.com/cosmos/gogoproto/blob/main/gogoproto/gogo.proto
File "gogoproto/gogo.proto" is used in ClickHouse to compile prometheus protobufs.

145
contrib/prometheus-protobufs-gogo/gogoproto/gogo.proto Normal file
View File

@ -0,0 +1,145 @@
// Protocol Buffers for Go with Gadgets
//
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
// http://github.com/cosmos/gogoproto
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
syntax = "proto2";
package gogoproto;
import "google/protobuf/descriptor.proto";
option java_package = "com.google.protobuf";
option java_outer_classname = "GoGoProtos";
option go_package = "github.com/cosmos/gogoproto/gogoproto";
extend google.protobuf.EnumOptions {
optional bool goproto_enum_prefix = 62001;
optional bool goproto_enum_stringer = 62021;
optional bool enum_stringer = 62022;
optional string enum_customname = 62023;
optional bool enumdecl = 62024;
}
extend google.protobuf.EnumValueOptions {
optional string enumvalue_customname = 66001;
}
extend google.protobuf.FileOptions {
optional bool goproto_getters_all = 63001;
optional bool goproto_enum_prefix_all = 63002;
optional bool goproto_stringer_all = 63003;
optional bool verbose_equal_all = 63004;
optional bool face_all = 63005;
optional bool gostring_all = 63006;
optional bool populate_all = 63007;
optional bool stringer_all = 63008;
optional bool onlyone_all = 63009;
optional bool equal_all = 63013;
optional bool description_all = 63014;
optional bool testgen_all = 63015;
optional bool benchgen_all = 63016;
optional bool marshaler_all = 63017;
optional bool unmarshaler_all = 63018;
optional bool stable_marshaler_all = 63019;
optional bool sizer_all = 63020;
optional bool goproto_enum_stringer_all = 63021;
optional bool enum_stringer_all = 63022;
optional bool unsafe_marshaler_all = 63023;
optional bool unsafe_unmarshaler_all = 63024;
optional bool goproto_extensions_map_all = 63025;
optional bool goproto_unrecognized_all = 63026;
optional bool gogoproto_import = 63027;
optional bool protosizer_all = 63028;
optional bool compare_all = 63029;
optional bool typedecl_all = 63030;
optional bool enumdecl_all = 63031;
optional bool goproto_registration = 63032;
optional bool messagename_all = 63033;
optional bool goproto_sizecache_all = 63034;
optional bool goproto_unkeyed_all = 63035;
}
extend google.protobuf.MessageOptions {
optional bool goproto_getters = 64001;
optional bool goproto_stringer = 64003;
optional bool verbose_equal = 64004;
optional bool face = 64005;
optional bool gostring = 64006;
optional bool populate = 64007;
optional bool stringer = 67008;
optional bool onlyone = 64009;
optional bool equal = 64013;
optional bool description = 64014;
optional bool testgen = 64015;
optional bool benchgen = 64016;
optional bool marshaler = 64017;
optional bool unmarshaler = 64018;
optional bool stable_marshaler = 64019;
optional bool sizer = 64020;
optional bool unsafe_marshaler = 64023;
optional bool unsafe_unmarshaler = 64024;
optional bool goproto_extensions_map = 64025;
optional bool goproto_unrecognized = 64026;
optional bool protosizer = 64028;
optional bool compare = 64029;
optional bool typedecl = 64030;
optional bool messagename = 64033;
optional bool goproto_sizecache = 64034;
optional bool goproto_unkeyed = 64035;
}
extend google.protobuf.FieldOptions {
optional bool nullable = 65001;
optional bool embed = 65002;
optional string customtype = 65003;
optional string customname = 65004;
optional string jsontag = 65005;
optional string moretags = 65006;
optional string casttype = 65007;
optional string castkey = 65008;
optional string castvalue = 65009;
optional bool stdtime = 65010;
optional bool stdduration = 65011;
optional bool wktpointer = 65012;
optional string castrepeated = 65013;
}

201
contrib/prometheus-protobufs/LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

2
contrib/prometheus-protobufs/README Normal file
View File

@ -0,0 +1,2 @@
Files "prompb/remote.proto" and "prompb/types.proto" were downloaded from the Prometheus repository:
https://github.com/prometheus/prometheus/tree/main/prompb

88
contrib/prometheus-protobufs/prompb/remote.proto Normal file
View File

@ -0,0 +1,88 @@
// Copyright 2016 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package prometheus;
option go_package = "prompb";
import "prompb/types.proto";
import "gogoproto/gogo.proto";
message WriteRequest {
repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false];
// Cortex uses this field to determine the source of the write request.
// We reserve it to avoid any compatibility issues.
reserved 2;
repeated prometheus.MetricMetadata metadata = 3 [(gogoproto.nullable) = false];
}
// ReadRequest represents a remote read request.
message ReadRequest {
repeated Query queries = 1;
enum ResponseType {
// Server will return a single ReadResponse message with matched series that includes list of raw samples.
// It's recommended to use streamed response types instead.
//
// Response headers:
// Content-Type: "application/x-protobuf"
// Content-Encoding: "snappy"
SAMPLES = 0;
// Server will stream a delimited ChunkedReadResponse message that
// contains XOR or HISTOGRAM(!) encoded chunks for a single series.
// Each message is following varint size and fixed size bigendian
// uint32 for CRC32 Castagnoli checksum.
//
// Response headers:
// Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse"
// Content-Encoding: ""
STREAMED_XOR_CHUNKS = 1;
}
// accepted_response_types allows negotiating the content type of the response.
//
// Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is
// implemented by server, error is returned.
// For request that do not contain `accepted_response_types` field the SAMPLES response type will be used.
repeated ResponseType accepted_response_types = 2;
}
// ReadResponse is a response when response_type equals SAMPLES.
message ReadResponse {
// In same order as the request's queries.
repeated QueryResult results = 1;
}
message Query {
int64 start_timestamp_ms = 1;
int64 end_timestamp_ms = 2;
repeated prometheus.LabelMatcher matchers = 3;
prometheus.ReadHints hints = 4;
}
message QueryResult {
// Samples within a time series must be ordered by time.
repeated prometheus.TimeSeries timeseries = 1;
}
// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS.
// We strictly stream full series after series, optionally split by time. This means that a single frame can contain
// partition of the single series, but once a new series is started to be streamed it means that no more chunks will
// be sent for previous one. Series are returned sorted in the same way TSDB block are internally.
message ChunkedReadResponse {
repeated prometheus.ChunkedSeries chunked_series = 1;
// query_index represents an index of the query from ReadRequest.queries these chunks relates to.
int64 query_index = 2;
}

187
contrib/prometheus-protobufs/prompb/types.proto Normal file
View File

@ -0,0 +1,187 @@
// Copyright 2017 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package prometheus;
option go_package = "prompb";
import "gogoproto/gogo.proto";
message MetricMetadata {
enum MetricType {
UNKNOWN = 0;
COUNTER = 1;
GAUGE = 2;
HISTOGRAM = 3;
GAUGEHISTOGRAM = 4;
SUMMARY = 5;
INFO = 6;
STATESET = 7;
}
// Represents the metric type, these match the set from Prometheus.
// Refer to github.com/prometheus/common/model/metadata.go for details.
MetricType type = 1;
string metric_family_name = 2;
string help = 4;
string unit = 5;
}
message Sample {
double value = 1;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 2;
}
message Exemplar {
// Optional, can be empty.
repeated Label labels = 1 [(gogoproto.nullable) = false];
double value = 2;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 3;
}
// A native histogram, also known as a sparse histogram.
// Original design doc:
// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit
// The appendix of this design doc also explains the concept of float
// histograms. This Histogram message can represent both, the usual
// integer histogram as well as a float histogram.
message Histogram {
enum ResetHint {
UNKNOWN = 0; // Need to test for a counter reset explicitly.
YES = 1; // This is the 1st histogram after a counter reset.
NO = 2; // There was no counter reset between this and the previous Histogram.
GAUGE = 3; // This is a gauge histogram where counter resets don't happen.
}
oneof count { // Count of observations in the histogram.
uint64 count_int = 1;
double count_float = 2;
}
double sum = 3; // Sum of observations in the histogram.
// The schema defines the bucket schema. Currently, valid numbers
// are -4 <= n <= 8. They are all for base-2 bucket schemas, where 1
// is a bucket boundary in each case, and then each power of two is
// divided into 2^n logarithmic buckets. Or in other words, each
// bucket boundary is the previous boundary times 2^(2^-n). In the
// future, more bucket schemas may be added using numbers < -4 or >
// 8.
sint32 schema = 4;
double zero_threshold = 5; // Breadth of the zero bucket.
oneof zero_count { // Count in zero bucket.
uint64 zero_count_int = 6;
double zero_count_float = 7;
}
// Negative Buckets.
repeated BucketSpan negative_spans = 8 [(gogoproto.nullable) = false];
// Use either "negative_deltas" or "negative_counts", the former for
// regular histograms with integer counts, the latter for float
// histograms.
repeated sint64 negative_deltas = 9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
repeated double negative_counts = 10; // Absolute count of each bucket.
// Positive Buckets.
repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false];
// Use either "positive_deltas" or "positive_counts", the former for
// regular histograms with integer counts, the latter for float
// histograms.
repeated sint64 positive_deltas = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
repeated double positive_counts = 13; // Absolute count of each bucket.
ResetHint reset_hint = 14;
// timestamp is in ms format, see model/timestamp/timestamp.go for
// conversion from time.Time to Prometheus timestamp.
int64 timestamp = 15;
}
// A BucketSpan defines a number of consecutive buckets with their
// offset. Logically, it would be more straightforward to include the
// bucket counts in the Span. However, the protobuf representation is
// more compact in the way the data is structured here (with all the
// buckets in a single array separate from the Spans).
message BucketSpan {
sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative).
uint32 length = 2; // Length of consecutive buckets.
}
// TimeSeries represents samples and labels for a single time series.
message TimeSeries {
// For a timeseries to be valid, and for the samples and exemplars
// to be ingested by the remote system properly, the labels field is required.
repeated Label labels = 1 [(gogoproto.nullable) = false];
repeated Sample samples = 2 [(gogoproto.nullable) = false];
repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false];
repeated Histogram histograms = 4 [(gogoproto.nullable) = false];
}
message Label {
string name = 1;
string value = 2;
}
message Labels {
repeated Label labels = 1 [(gogoproto.nullable) = false];
}
// Matcher specifies a rule, which can match or set of labels or not.
message LabelMatcher {
enum Type {
EQ = 0;
NEQ = 1;
RE = 2;
NRE = 3;
}
Type type = 1;
string name = 2;
string value = 3;
}
message ReadHints {
int64 step_ms = 1; // Query step size in milliseconds.
string func = 2; // String representation of surrounding function or aggregation.
int64 start_ms = 3; // Start time in milliseconds.
int64 end_ms = 4; // End time in milliseconds.
repeated string grouping = 5; // List of label names used in aggregation.
bool by = 6; // Indicate whether it is without or by.
int64 range_ms = 7; // Range vector selector range in milliseconds.
}
// Chunk represents a TSDB chunk.
// Time range [min, max] is inclusive.
message Chunk {
int64 min_time_ms = 1;
int64 max_time_ms = 2;
// We require this to match chunkenc.Encoding.
enum Encoding {
UNKNOWN = 0;
XOR = 1;
HISTOGRAM = 2;
FLOAT_HISTOGRAM = 3;
}
Encoding type = 3;
bytes data = 4;
}
// ChunkedSeries represents single, encoded time series.
message ChunkedSeries {
// Labels should be sorted.
repeated Label labels = 1 [(gogoproto.nullable) = false];
// Chunks will be in start time order and may overlap.
repeated Chunk chunks = 2 [(gogoproto.nullable) = false];
}

2
contrib/s2geometry vendored

@ -1 +1 @@
Subproject commit 0547c38371777a1c1c8be263a6f05c3bf71bb05b
Subproject commit 6522a40338d58752c2a4227a3fc2bc4107c73e43

19
contrib/s2geometry-cmake/CMakeLists.txt
View File

@ -1,7 +1,7 @@
option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES})
option(ENABLE_S2_GEOMETRY "Enable S2 Geometry" ${ENABLE_LIBRARIES})
if (NOT ENABLE_S2_GEOMETRY)
message(STATUS "Not using S2 geometry")
message(STATUS "Not using S2 Geometry")
return()
endif()
@ -38,6 +38,7 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2cell_index.cc"
"${S2_SOURCE_DIR}/s2/s2cell_union.cc"
"${S2_SOURCE_DIR}/s2/s2centroids.cc"
"${S2_SOURCE_DIR}/s2/s2chain_interpolation_query.cc"
"${S2_SOURCE_DIR}/s2/s2closest_cell_query.cc"
"${S2_SOURCE_DIR}/s2/s2closest_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2closest_point_query.cc"
@ -46,6 +47,7 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2coords.cc"
"${S2_SOURCE_DIR}/s2/s2crossing_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2debug.cc"
"${S2_SOURCE_DIR}/s2/s2density_tree.cc"
"${S2_SOURCE_DIR}/s2/s2earth.cc"
"${S2_SOURCE_DIR}/s2/s2edge_clipping.cc"
"${S2_SOURCE_DIR}/s2/s2edge_crosser.cc"
@ -53,8 +55,10 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2edge_distances.cc"
"${S2_SOURCE_DIR}/s2/s2edge_tessellator.cc"
"${S2_SOURCE_DIR}/s2/s2error.cc"
"${S2_SOURCE_DIR}/s2/s2fractal.cc"
"${S2_SOURCE_DIR}/s2/s2furthest_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2hausdorff_distance_query.cc"
"${S2_SOURCE_DIR}/s2/s2index_cell_data.cc"
"${S2_SOURCE_DIR}/s2/s2latlng.cc"
"${S2_SOURCE_DIR}/s2/s2latlng_rect.cc"
"${S2_SOURCE_DIR}/s2/s2latlng_rect_bounder.cc"
@ -63,10 +67,10 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2lax_polyline_shape.cc"
"${S2_SOURCE_DIR}/s2/s2loop.cc"
"${S2_SOURCE_DIR}/s2/s2loop_measures.cc"
"${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2measures.cc"
"${S2_SOURCE_DIR}/s2/s2memory_tracker.cc"
"${S2_SOURCE_DIR}/s2/s2metrics.cc"
"${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2min_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2padded_cell.cc"
"${S2_SOURCE_DIR}/s2/s2point_compression.cc"
@ -80,10 +84,11 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2predicates.cc"
"${S2_SOURCE_DIR}/s2/s2projections.cc"
"${S2_SOURCE_DIR}/s2/s2r2rect.cc"
"${S2_SOURCE_DIR}/s2/s2region.cc"
"${S2_SOURCE_DIR}/s2/s2region_term_indexer.cc"
"${S2_SOURCE_DIR}/s2/s2random.cc"
"${S2_SOURCE_DIR}/s2/s2region_coverer.cc"
"${S2_SOURCE_DIR}/s2/s2region_intersection.cc"
"${S2_SOURCE_DIR}/s2/s2region_sharder.cc"
"${S2_SOURCE_DIR}/s2/s2region_term_indexer.cc"
"${S2_SOURCE_DIR}/s2/s2region_union.cc"
"${S2_SOURCE_DIR}/s2/s2shape_index.cc"
"${S2_SOURCE_DIR}/s2/s2shape_index_buffered_region.cc"
@ -94,9 +99,12 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2shapeutil_coding.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_contains_brute_force.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_conversion.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_count_vertices.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_edge_iterator.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_edge_wrap.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_get_reference_point.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_visit_crossing_edge_pairs.cc"
"${S2_SOURCE_DIR}/s2/s2testing.cc"
"${S2_SOURCE_DIR}/s2/s2text_format.cc"
"${S2_SOURCE_DIR}/s2/s2wedge_relations.cc"
"${S2_SOURCE_DIR}/s2/s2winding_operation.cc"
@ -140,6 +148,7 @@ target_link_libraries(_s2 PRIVATE
absl::strings
absl::type_traits
absl::utility
absl::vlog_is_on
)
target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")

2
contrib/vectorscan vendored

@ -1 +1 @@
Subproject commit 4918f81ea3d1abd18905bac9876d4a1fe2ebdf07
Subproject commit d29730e1cb9daaa66bda63426cdce83505d2c809

27
contrib/vectorscan-cmake/CMakeLists.txt
View File

@ -1,11 +1,8 @@
# We use vectorscan, a portable and API/ABI-compatible drop-in replacement for hyperscan.
# Vectorscan is drop-in replacement for Hyperscan.
if ((ARCH_AMD64 AND NOT NO_SSE3_OR_HIGHER) OR ARCH_AARCH64)
option (ENABLE_VECTORSCAN "Enable vectorscan library" ${ENABLE_LIBRARIES})
option (ENABLE_VECTORSCAN "Enable vectorscan" ${ENABLE_LIBRARIES})
endif()
# TODO PPC should generally work but needs manual generation of ppc/config.h file on a PPC machine
if (NOT ENABLE_VECTORSCAN)
message (STATUS "Not using vectorscan")
return()
@ -272,34 +269,24 @@ if (ARCH_AARCH64)
)
endif()
# TODO
# if (ARCH_PPC64LE)
# list(APPEND SRCS
# "${LIBRARY_DIR}/src/util/supervector/arch/ppc64el/impl.cpp"
# )
# endif()
add_library (_vectorscan ${SRCS})
target_compile_options (_vectorscan PRIVATE
-fno-sanitize=undefined # assume the library takes care of itself
-O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # options from original build system
)
# library has too much debug information
if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options (_vectorscan PRIVATE -g0)
endif()
# Include version header manually generated by running the original build system
target_include_directories (_vectorscan SYSTEM PRIVATE common)
target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src")
# Makes the version header visible. It was generated by running the native build system manually.
# Please update whenever you update vectorscan.
target_include_directories (_vectorscan SYSTEM PUBLIC common)
# vectorscan inherited some patched in-source versions of boost headers to fix a bug in
# boost 1.69. This bug has been solved long ago but vectorscan's source code still
# points to the patched versions, so include it here.
target_include_directories (_vectorscan SYSTEM PRIVATE "${LIBRARY_DIR}/include")
target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src")
# Include platform-specific config header generated by manually running the original build system
# Please regenerate these files if you update vectorscan.

6
contrib/vectorscan-cmake/common/hs_version.h
View File

@ -32,8 +32,12 @@
/**
* A version string to identify this release of Hyperscan.
*/
#define HS_VERSION_STRING "5.4.7 2022-06-20"
#define HS_VERSION_STRING "5.4.11 2024-07-04"
#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0)
#define HS_MAJOR 5
#define HS_MINOR 4
#define HS_PATCH 11
#endif /* HS_VERSION_H_C6428FAF8E3713 */

1
docs/en/operations/backup.md
View File

@ -84,6 +84,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
- [`compression_method`](/docs/en/sql-reference/statements/create/table.md/#column-compression-codecs) and compression_level
- `password` for the file on disk
- `base_backup`: the destination of the previous backup of this source. For example, `Disk('backups', '1.zip')`
- `use_same_s3_credentials_for_base_backup`: whether base backup to S3 should inherit credentials from the query. Only works with `S3`.
- `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables
- `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family.
- `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD`

36
docs/en/sql-reference/table-functions/fuzzQuery.md Normal file
View File

@ -0,0 +1,36 @@
---
slug: /en/sql-reference/table-functions/fuzzQuery
sidebar_position: 75
sidebar_label: fuzzQuery
---
# fuzzQuery
Perturbs the given query string with random variations.
``` sql
fuzzQuery(query[, max_query_length[, random_seed]])
```
**Arguments**
- `query` (String) - The source query to perform the fuzzing on.
- `max_query_length` (UInt64) - A maximum length the query can get during the fuzzing process.
- `random_seed` (UInt64) - A random seed for producing stable results.
**Returned Value**
A table object with a single column containing perturbed query strings.
## Usage Example
``` sql
SELECT * FROM fuzzQuery('SELECT materialize(\'a\' AS key) GROUP BY key') LIMIT 2;
```
```
┌─query──────────────────────────────────────────────────────────┐
1. │ SELECT 'a' AS key GROUP BY key │
2. │ EXPLAIN PIPELINE compact = true SELECT 'a' AS key GROUP BY key │
└────────────────────────────────────────────────────────────────┘
```

1
docs/en/sql-reference/window-functions/index.md
View File

@ -23,6 +23,7 @@ ClickHouse supports the standard grammar for defining windows and window functio
| `GROUPS` frame | ❌ |
| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | ✅ (All aggregate functions are supported) |
| `rank()`, `dense_rank()`, `row_number()` | ✅ |
| `percent_rank()` | ✅ Efficiently computes the relative standing of a value within a partition in a dataset. This function effectively replaces the more verbose and computationally intensive manual SQL calculation expressed as `ifNull((rank() OVER(PARTITION BY x ORDER BY y) - 1) / nullif(count(1) OVER(PARTITION BY x) - 1, 0), 0)`|
| `lag/lead(value, offset)` | ❌ <br/> You can use one of the following workarounds:<br/> 1) `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead` <br/> 2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` |
| ntile(buckets) | ✅ <br/> Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |

5
programs/client/Client.h
View File

@ -9,7 +9,10 @@ namespace DB
class Client : public ClientBase
{
public:
Client() = default;
Client()
{
fuzzer = QueryFuzzer(randomSeed(), &std::cout, &std::cerr);
}
void initialize(Poco::Util::Application & self) override;

5
programs/keeper/Keeper.cpp
View File

@ -27,6 +27,8 @@
#include <sys/stat.h>
#include <pwd.h>
#include <Common/Jemalloc.h>
#include <Interpreters/Context.h>
#include <Coordination/FourLetterCommand.h>
@ -267,6 +269,9 @@ HTTPContextPtr httpContext()
int Keeper::main(const std::vector<std::string> & /*args*/)
try
{
#if USE_JEMALLOC
setJemallocBackgroundThreads(true);
#endif
Poco::Logger * log = &logger();
UseSSL use_ssl;

5
programs/server/Server.cpp
View File

@ -11,6 +11,7 @@
#include <Poco/Util/HelpFormatter.h>
#include <Poco/Environment.h>
#include <Poco/Config.h>
#include <Common/Jemalloc.h>
#include <Common/scope_guard_safe.h>
#include <Common/logger_useful.h>
#include <base/phdr_cache.h>
@ -628,6 +629,10 @@ static void initializeAzureSDKLogger(
int Server::main(const std::vector<std::string> & /*args*/)
try
{
#if USE_JEMALLOC
setJemallocBackgroundThreads(true);
#endif
Stopwatch startup_watch;
Poco::Logger * log = &logger();

116
src/Backups/BackupIO_AzureBlobStorage.cpp
View File

@ -29,48 +29,49 @@ namespace ErrorCodes
}
BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
const ContextPtr & context_)
: BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage"))
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
, configuration(configuration_)
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false}
, connection_params(connection_params_)
, blob_path(blob_path_)
{
auto client_ptr = configuration.createClient(/* is_readonly */false, /* attempt_to_create_container */true);
client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);
auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_->getSettingsRef(), allow_azure_native_copy);
object_storage = std::make_unique<AzureObjectStorage>("BackupReaderAzureBlobStorage",
object_storage = std::make_unique<AzureObjectStorage>(
"BackupReaderAzureBlobStorage",
std::move(client_ptr),
configuration.createSettings(context_),
configuration_.container,
configuration.getConnectionURL().toString());
std::move(settings_ptr),
connection_params.getContainer(),
connection_params.getConnectionURL());
client = object_storage->getAzureBlobStorageClient();
auto settings_copy = *object_storage->getSettings();
settings_copy.use_native_copy = allow_azure_native_copy;
settings = std::make_unique<const AzureObjectStorageSettings>(settings_copy);
settings = object_storage->getSettings();
}
BackupReaderAzureBlobStorage::~BackupReaderAzureBlobStorage() = default;
bool BackupReaderAzureBlobStorage::fileExists(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return object_storage->exists(StoredObject(key));
}
UInt64 BackupReaderAzureBlobStorage::getFileSize(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
ObjectMetadata object_metadata = object_storage->getObjectMetadata(key);
return object_metadata.size_bytes;
}
std::unique_ptr<SeekableReadBuffer> BackupReaderAzureBlobStorage::readFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<ReadBufferFromAzureBlobStorage>(
client, key, read_settings, settings->max_single_read_retries,
settings->max_single_download_retries);
@ -85,23 +86,23 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup,
&& destination_data_source_description.is_encrypted == encrypted_in_backup)
{
LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName());
auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
auto write_blob_function = [&](const Strings & dst_blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
{
/// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
if (blob_path.size() != 2 || mode != WriteMode::Rewrite)
if (dst_blob_path.size() != 2 || mode != WriteMode::Rewrite)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Blob writing function called with unexpected blob_path.size={} or mode={}",
blob_path.size(), mode);
dst_blob_path.size(), mode);
copyAzureBlobStorageFile(
client,
destination_disk->getObjectStorage()->getAzureBlobStorageClient(),
configuration.container,
fs::path(configuration.blob_path) / path_in_backup,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
0,
file_size,
/* dest_container */ blob_path[1],
/* dest_path */ blob_path[0],
/* dest_container */ dst_blob_path[1],
/* dest_path */ dst_blob_path[0],
settings,
read_settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupRDAzure"));
@ -119,28 +120,33 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup,
BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
const ContextPtr & context_,
bool attempt_to_create_container)
: BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage"))
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
, configuration(configuration_)
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false}
, connection_params(connection_params_)
, blob_path(blob_path_)
{
auto client_ptr = configuration.createClient(/* is_readonly */false, attempt_to_create_container);
client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
if (!attempt_to_create_container)
connection_params.endpoint.container_already_exists = true;
object_storage = std::make_unique<AzureObjectStorage>("BackupWriterAzureBlobStorage",
auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);
auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_->getSettingsRef(), allow_azure_native_copy);
object_storage = std::make_unique<AzureObjectStorage>(
"BackupWriterAzureBlobStorage",
std::move(client_ptr),
configuration.createSettings(context_),
configuration.container,
configuration_.getConnectionURL().toString());
std::move(settings_ptr),
connection_params.getContainer(),
connection_params.getConnectionURL());
client = object_storage->getAzureBlobStorageClient();
auto settings_copy = *object_storage->getSettings();
settings_copy.use_native_copy = allow_azure_native_copy;
settings = std::make_unique<const AzureObjectStorageSettings>(settings_copy);
settings = object_storage->getSettings();
}
void BackupWriterAzureBlobStorage::copyFileFromDisk(
@ -159,18 +165,18 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(
{
/// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in AzureBlobStorage container.
/// In this case we can't use the native copy.
if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2)
if (auto src_blob_path = src_disk->getBlobPath(src_path); src_blob_path.size() == 2)
{
LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorag", src_path, src_disk->getName());
copyAzureBlobStorageFile(
src_disk->getObjectStorage()->getAzureBlobStorageClient(),
client,
/* src_container */ blob_path[1],
/* src_path */ blob_path[0],
/* src_container */ src_blob_path[1],
/* src_path */ src_blob_path[0],
start_pos,
length,
configuration.container,
fs::path(configuration.blob_path) / path_in_backup,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
settings,
read_settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
@ -188,11 +194,11 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St
copyAzureBlobStorageFile(
client,
client,
configuration.container,
fs::path(configuration.blob_path)/ source,
connection_params.getContainer(),
fs::path(blob_path)/ source,
0,
size,
/* dest_container */ configuration.container,
/* dest_container */ connection_params.getContainer(),
/* dest_path */ destination,
settings,
read_settings,
@ -206,22 +212,28 @@ void BackupWriterAzureBlobStorage::copyDataToFile(
UInt64 length)
{
copyDataToAzureBlobStorageFile(
create_read_buffer, start_pos, length, client, configuration.container,
fs::path(configuration.blob_path) / path_in_backup, settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
create_read_buffer,
start_pos,
length,
client,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(),
"BackupWRAzure"));
}
BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default;
bool BackupWriterAzureBlobStorage::fileExists(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return object_storage->exists(StoredObject(key));
}
UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
RelativePathsWithMetadata children;
object_storage->listObjects(key,children,/*max_keys*/0);
if (children.empty())
@ -231,7 +243,7 @@ UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name)
std::unique_ptr<ReadBuffer> BackupWriterAzureBlobStorage::readFile(const String & file_name, size_t /*expected_file_size*/)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<ReadBufferFromAzureBlobStorage>(
client, key, read_settings, settings->max_single_read_retries,
settings->max_single_download_retries);
@ -239,7 +251,7 @@ std::unique_ptr<ReadBuffer> BackupWriterAzureBlobStorage::readFile(const String
std::unique_ptr<WriteBuffer> BackupWriterAzureBlobStorage::writeFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<WriteBufferFromAzureBlobStorage>(
client,
key,
@ -251,7 +263,7 @@ std::unique_ptr<WriteBuffer> BackupWriterAzureBlobStorage::writeFile(const Strin
void BackupWriterAzureBlobStorage::removeFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
StoredObject object(key);
object_storage->removeObjectIfExists(object);
}
@ -260,7 +272,7 @@ void BackupWriterAzureBlobStorage::removeFiles(const Strings & file_names)
{
StoredObjects objects;
for (const auto & file_name : file_names)
objects.emplace_back(fs::path(configuration.blob_path) / file_name);
objects.emplace_back(fs::path(blob_path) / file_name);
object_storage->removeObjectsIfExist(objects);
@ -270,7 +282,7 @@ void BackupWriterAzureBlobStorage::removeFilesBatch(const Strings & file_names)
{
StoredObjects objects;
for (const auto & file_name : file_names)
objects.emplace_back(fs::path(configuration.blob_path) / file_name);
objects.emplace_back(fs::path(blob_path) / file_name);
object_storage->removeObjectsIfExist(objects);
}

20
src/Backups/BackupIO_AzureBlobStorage.h
View File

@ -1,12 +1,10 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_Default.h>
#include <Disks/DiskType.h>
#include <Interpreters/Context_fwd.h>
#include <Storages/ObjectStorage/Azure/Configuration.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
namespace DB
@ -17,7 +15,8 @@ class BackupReaderAzureBlobStorage : public BackupReaderDefault
{
public:
BackupReaderAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
@ -40,16 +39,18 @@ public:
private:
const DataSourceDescription data_source_description;
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
StorageAzureConfiguration configuration;
AzureBlobStorage::ConnectionParams connection_params;
String blob_path;
std::unique_ptr<AzureObjectStorage> object_storage;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
};
class BackupWriterAzureBlobStorage : public BackupWriterDefault
{
public:
BackupWriterAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
@ -87,9 +88,10 @@ private:
const DataSourceDescription data_source_description;
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
StorageAzureConfiguration configuration;
AzureBlobStorage::ConnectionParams connection_params;
String blob_path;
std::unique_ptr<AzureObjectStorage> object_storage;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
};
}

68
src/Backups/registerBackupEngineAzureBlobStorage.cpp
View File

@ -5,6 +5,7 @@
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_AzureBlobStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Backups/BackupImpl.h>
#include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <Interpreters/Context.h>
@ -49,7 +50,9 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
const String & id_arg = params.backup_info.id_arg;
const auto & args = params.backup_info.args;
StorageAzureConfiguration configuration;
String blob_path;
AzureBlobStorage::ConnectionParams connection_params;
auto request_settings = AzureBlobStorage::getRequestSettings(params.context->getSettingsRef());
if (!id_arg.empty())
{
@ -59,55 +62,42 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
if (!config.has(config_prefix))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", id_arg);
if (config.has(config_prefix + ".connection_string"))
connection_params =
{
configuration.connection_url = config.getString(config_prefix + ".connection_string");
configuration.is_connection_string = true;
configuration.container = config.getString(config_prefix + ".container");
}
else
{
configuration.connection_url = config.getString(config_prefix + ".storage_account_url");
configuration.is_connection_string = false;
configuration.container = config.getString(config_prefix + ".container");
configuration.account_name = config.getString(config_prefix + ".account_name");
configuration.account_key = config.getString(config_prefix + ".account_key");
if (config.has(config_prefix + ".account_name") && config.has(config_prefix + ".account_key"))
{
configuration.account_name = config.getString(config_prefix + ".account_name");
configuration.account_key = config.getString(config_prefix + ".account_key");
}
}
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true),
};
if (args.size() > 1)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Backup AzureBlobStorage requires 1 or 2 arguments: named_collection, [filename]");
if (args.size() == 1)
configuration.setPath(args[0].safeGet<String>());
blob_path = args[0].safeGet<String>();
}
else
{
if (args.size() == 3)
{
configuration.connection_url = args[0].safeGet<String>();
configuration.is_connection_string = !configuration.connection_url.starts_with("http");
auto connection_url = args[0].safeGet<String>();
auto container_name = args[1].safeGet<String>();
blob_path = args[2].safeGet<String>();
configuration.container = args[1].safeGet<String>();
configuration.blob_path = args[2].safeGet<String>();
AzureBlobStorage::processURL(connection_url, container_name, connection_params.endpoint, connection_params.auth_method);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true);
}
else if (args.size() == 5)
{
configuration.connection_url = args[0].safeGet<String>();
configuration.is_connection_string = false;
connection_params.endpoint.storage_account_url = args[0].safeGet<String>();
connection_params.endpoint.container_name = args[1].safeGet<String>();
blob_path = args[2].safeGet<String>();
configuration.container = args[1].safeGet<String>();
configuration.blob_path = args[2].safeGet<String>();
configuration.account_name = args[3].safeGet<String>();
configuration.account_key = args[4].safeGet<String>();
auto account_name = args[3].safeGet<String>();
auto account_key = args[4].safeGet<String>();
connection_params.auth_method = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(account_name, account_key);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true);
}
else
{
@ -117,16 +107,12 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
}
BackupImpl::ArchiveParams archive_params;
if (hasRegisteredArchiveFileExtension(configuration.getPath()))
if (hasRegisteredArchiveFileExtension(blob_path))
{
if (params.is_internal_backup)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Using archives with backups on clusters is disabled");
auto path = configuration.getPath();
auto filename = removeFileNameFromURL(path);
configuration.setPath(path);
archive_params.archive_name = filename;
archive_params.archive_name = removeFileNameFromURL(blob_path);
archive_params.compression_method = params.compression_method;
archive_params.compression_level = params.compression_level;
archive_params.password = params.password;
@ -141,7 +127,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
if (params.open_mode == IBackup::OpenMode::READ)
{
auto reader = std::make_shared<BackupReaderAzureBlobStorage>(
configuration,
connection_params,
blob_path,
params.allow_azure_native_copy,
params.read_settings,
params.write_settings,
@ -159,7 +146,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
else
{
auto writer = std::make_shared<BackupWriterAzureBlobStorage>(
configuration,
connection_params,
blob_path,
params.allow_azure_native_copy,
params.read_settings,
params.write_settings,

4
src/CMakeLists.txt
View File

@ -607,6 +607,10 @@ if (TARGET ch_contrib::usearch)
dbms_target_link_libraries(PUBLIC ch_contrib::usearch)
endif()
if (TARGET ch_contrib::prometheus_protobufs)
dbms_target_link_libraries (PUBLIC ch_contrib::prometheus_protobufs)
endif()
if (TARGET ch_rust::skim)
dbms_target_include_directories(PRIVATE $<TARGET_PROPERTY:ch_rust::skim,INTERFACE_INCLUDE_DIRECTORIES>)
dbms_target_link_libraries(PUBLIC ch_rust::skim)

2
src/Client/ClientBase.h
View File

@ -6,13 +6,13 @@
#include <Common/ProgressIndication.h>
#include <Common/InterruptListener.h>
#include <Common/ShellCommand.h>
#include <Common/QueryFuzzer.h>
#include <Common/Stopwatch.h>
#include <Common/DNSResolver.h>
#include <Core/ExternalTable.h>
#include <Poco/Util/Application.h>
#include <Interpreters/Context.h>
#include <Client/Suggest.h>
#include <Client/QueryFuzzer.h>
#include <boost/program_options.hpp>
#include <Storages/StorageFile.h>
#include <Storages/SelectQueryInfo.h>

6
src/Client/HedgedConnections.cpp
View File

@ -195,6 +195,12 @@ void HedgedConnections::sendQuery(
modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
}
/// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
/// Make the analyzer being set, so it will be effectively applied on the remote server.
/// In other words, the initiator always controls whether the analyzer enabled or not for
/// all servers involved in the distributed query processing.
modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
replica.connection->sendQuery(timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout);
replica.packet_receiver->setTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout);

6
src/Client/MultiplexedConnections.cpp
View File

@ -150,6 +150,12 @@ void MultiplexedConnections::sendQuery(
}
}
/// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
/// Make the analyzer being set, so it will be effectively applied on the remote server.
/// In other words, the initiator always controls whether the analyzer enabled or not for
/// all servers involved in the distributed query processing.
modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && settings.allow_experimental_parallel_reading_from_replicas == 0;
size_t num_replicas = replica_states.size();

313
src/Common/CgroupsMemoryUsageObserver.cpp
View File

@ -12,7 +12,9 @@
#include <base/getMemoryAmount.h>
#include <base/sleep.h>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <optional>
#include "config.h"
@ -22,24 +24,169 @@
#define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif
using namespace DB;
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_CLOSE_FILE;
extern const int CANNOT_OPEN_FILE;
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_DATA;
}
CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_)
: log(getLogger("CgroupsMemoryUsageObserver"))
, wait_time(wait_time_)
, memory_usage_file(log)
}
namespace
{
LOG_INFO(log, "Initialized cgroups memory limit observer, wait time is {} sec", wait_time.count());
/// Format is
/// kernel 5
/// rss 15
/// [...]
uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key)
{
while (!buf.eof())
{
std::string current_key;
readStringUntilWhitespace(current_key, buf);
if (current_key != key)
{
std::string dummy;
readStringUntilNewlineInto(dummy, buf);
buf.ignore();
continue;
}
assertChar(' ', buf);
uint64_t value = 0;
readIntText(value, buf);
return value;
}
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find '{}' in '{}'", key, buf.getFileName());
}
struct CgroupsV1Reader : ICgroupsReader
{
explicit CgroupsV1Reader(const std::filesystem::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { }
uint64_t readMemoryUsage() override
{
std::lock_guard lock(mutex);
buf.rewind();
return readMetricFromStatFile(buf, "rss");
}
private:
std::mutex mutex;
ReadBufferFromFile buf TSA_GUARDED_BY(mutex);
};
struct CgroupsV2Reader : ICgroupsReader
{
explicit CgroupsV2Reader(const std::filesystem::path & stat_file_dir)
: current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat")
{
}
uint64_t readMemoryUsage() override
{
std::lock_guard lock(mutex);
current_buf.rewind();
stat_buf.rewind();
int64_t mem_usage = 0;
/// memory.current contains a single number
/// the reason why we subtract it described here: https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667
readIntText(mem_usage, current_buf);
mem_usage -= readMetricFromStatFile(stat_buf, "inactive_file");
chassert(mem_usage >= 0, "Negative memory usage");
return mem_usage;
}
private:
std::mutex mutex;
ReadBufferFromFile current_buf TSA_GUARDED_BY(mutex);
ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex);
};
/// Caveats:
/// - All of the logic in this file assumes that the current process is the only process in the
/// containing cgroup (or more precisely: the only process with significant memory consumption).
/// If this is not the case, then other processe's memory consumption may affect the internal
/// memory tracker ...
/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a
/// decade and will go away at some point, hierarchical detection is only implemented for v2.
/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such
/// systems existed only for a short transition period.
std::optional<std::string> getCgroupsV2Path()
{
if (!cgroupsV2Enabled())
return {};
if (!cgroupsV2MemoryControllerEnabled())
return {};
String cgroup = cgroupV2OfProcess();
auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup);
/// Return the bottom-most nested current memory file. If there is no such file at the current
/// level, try again at the parent level as memory settings are inherited.
while (current_cgroup != default_cgroups_mount.parent_path())
{
const auto current_path = current_cgroup / "memory.current";
const auto stat_path = current_cgroup / "memory.stat";
if (std::filesystem::exists(current_path) && std::filesystem::exists(stat_path))
return {current_cgroup};
current_cgroup = current_cgroup.parent_path();
}
return {};
}
std::optional<std::string> getCgroupsV1Path()
{
auto path = default_cgroups_mount / "memory/memory.stat";
if (!std::filesystem::exists(path))
return {};
return {default_cgroups_mount / "memory"};
}
std::pair<std::string, CgroupsMemoryUsageObserver::CgroupsVersion> getCgroupsPath()
{
auto v2_path = getCgroupsV2Path();
if (v2_path.has_value())
return {*v2_path, CgroupsMemoryUsageObserver::CgroupsVersion::V2};
auto v1_path = getCgroupsV1Path();
if (v1_path.has_value())
return {*v1_path, CgroupsMemoryUsageObserver::CgroupsVersion::V1};
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file");
}
}
namespace DB
{
CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_)
: log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_)
{
const auto [cgroup_path, version] = getCgroupsPath();
if (version == CgroupsVersion::V2)
cgroup_reader = std::make_unique<CgroupsV2Reader>(cgroup_path);
else
cgroup_reader = std::make_unique<CgroupsV1Reader>(cgroup_path);
LOG_INFO(
log,
"Will read the current memory usage from '{}' (cgroups version: {}), wait time is {} sec",
cgroup_path,
(version == CgroupsVersion::V1) ? "v1" : "v2",
wait_time.count());
}
CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver()
@ -84,7 +231,8 @@ void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint
mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
# endif
/// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them.
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
uint64_t memory_usage = cgroup_reader->readMemoryUsage();
LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage));
MemoryTracker::setRSS(memory_usage, 0);
LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage));
@ -104,152 +252,6 @@ void CgroupsMemoryUsageObserver::setOnMemoryAmountAvailableChangedFn(OnMemoryAmo
on_memory_amount_available_changed = on_memory_amount_available_changed_;
}
namespace
{
/// Caveats:
/// - All of the logic in this file assumes that the current process is the only process in the
/// containing cgroup (or more precisely: the only process with significant memory consumption).
/// If this is not the case, then other processe's memory consumption may affect the internal
/// memory tracker ...
/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a
/// decade and will go away at some point, hierarchical detection is only implemented for v2.
/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such
/// systems existed only for a short transition period.
std::optional<std::string> getCgroupsV2FileName()
{
if (!cgroupsV2Enabled())
return {};
if (!cgroupsV2MemoryControllerEnabled())
return {};
String cgroup = cgroupV2OfProcess();
auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup);
/// Return the bottom-most nested current memory file. If there is no such file at the current
/// level, try again at the parent level as memory settings are inherited.
while (current_cgroup != default_cgroups_mount.parent_path())
{
auto path = current_cgroup / "memory.current";
if (std::filesystem::exists(path))
return {path};
current_cgroup = current_cgroup.parent_path();
}
return {};
}
std::optional<std::string> getCgroupsV1FileName()
{
auto path = default_cgroups_mount / "memory/memory.stat";
if (!std::filesystem::exists(path))
return {};
return {path};
}
std::pair<std::string, CgroupsMemoryUsageObserver::CgroupsVersion> getCgroupsFileName()
{
auto v2_file_name = getCgroupsV2FileName();
if (v2_file_name.has_value())
return {*v2_file_name, CgroupsMemoryUsageObserver::CgroupsVersion::V2};
auto v1_file_name = getCgroupsV1FileName();
if (v1_file_name.has_value())
return {*v1_file_name, CgroupsMemoryUsageObserver::CgroupsVersion::V1};
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file");
}
}
CgroupsMemoryUsageObserver::MemoryUsageFile::MemoryUsageFile(LoggerPtr log_)
: log(log_)
{
std::tie(file_name, version) = getCgroupsFileName();
LOG_INFO(log, "Will read the current memory usage from '{}' (cgroups version: {})", file_name, (version == CgroupsVersion::V1) ? "v1" : "v2");
fd = ::open(file_name.data(), O_RDONLY);
if (fd == -1)
ErrnoException::throwFromPath(
(errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE,
file_name, "Cannot open file '{}'", file_name);
}
CgroupsMemoryUsageObserver::MemoryUsageFile::~MemoryUsageFile()
{
assert(fd != -1);
if (::close(fd) != 0)
{
try
{
ErrnoException::throwFromPath(
ErrorCodes::CANNOT_CLOSE_FILE,
file_name, "Cannot close file '{}'", file_name);
}
catch (const ErrnoException &)
{
tryLogCurrentException(log, __PRETTY_FUNCTION__);
}
}
}
uint64_t CgroupsMemoryUsageObserver::MemoryUsageFile::readMemoryUsage() const
{
/// File read is probably not read is thread-safe, just to be sure
std::lock_guard lock(mutex);
ReadBufferFromFileDescriptor buf(fd);
buf.rewind();
uint64_t mem_usage = 0;
switch (version)
{
case CgroupsVersion::V1:
{
/// Format is
/// kernel 5
/// rss 15
/// [...]
std::string key;
bool found_rss = false;
while (!buf.eof())
{
readStringUntilWhitespace(key, buf);
if (key != "rss")
{
std::string dummy;
readStringUntilNewlineInto(dummy, buf);
buf.ignore();
continue;
}
assertChar(' ', buf);
readIntText(mem_usage, buf);
found_rss = true;
break;
}
if (!found_rss)
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find 'rss' in '{}'", file_name);
break;
}
case CgroupsVersion::V2:
{
readIntText(mem_usage, buf);
break;
}
}
LOG_TRACE(log, "Read current memory usage {} from cgroups", ReadableSize(mem_usage));
return mem_usage;
}
void CgroupsMemoryUsageObserver::startThread()
{
if (!thread.joinable())
@ -301,7 +303,8 @@ void CgroupsMemoryUsageObserver::runThread()
std::lock_guard<std::mutex> limit_lock(limit_mutex);
if (soft_limit > 0 && hard_limit > 0)
{
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
uint64_t memory_usage = cgroup_reader->readMemoryUsage();
LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage));
if (memory_usage > hard_limit)
{
if (last_memory_usage <= hard_limit)

27
src/Common/CgroupsMemoryUsageObserver.h
View File

@ -3,11 +3,19 @@
#include <Common/ThreadPool.h>
#include <chrono>
#include <memory>
#include <mutex>
namespace DB
{
struct ICgroupsReader
{
virtual ~ICgroupsReader() = default;
virtual uint64_t readMemoryUsage() = 0;
};
/// Does two things:
/// 1. Periodically reads the memory usage of the process from Linux cgroups.
/// You can specify soft or hard memory limits:
@ -61,27 +69,12 @@ private:
uint64_t last_memory_usage = 0; /// how much memory does the process use
uint64_t last_available_memory_amount; /// how much memory can the process use
/// Represents the cgroup virtual file that shows the memory consumption of the process's cgroup.
struct MemoryUsageFile
{
public:
explicit MemoryUsageFile(LoggerPtr log_);
~MemoryUsageFile();
uint64_t readMemoryUsage() const;
private:
LoggerPtr log;
mutable std::mutex mutex;
int fd TSA_GUARDED_BY(mutex) = -1;
CgroupsVersion version;
std::string file_name;
};
MemoryUsageFile memory_usage_file;
void stopThread();
void runThread();
std::unique_ptr<ICgroupsReader> cgroup_reader;
std::mutex thread_mutex;
std::condition_variable cond;
ThreadFromGlobalPool thread;

5
src/Common/GWPAsan.cpp
View File

@ -57,9 +57,12 @@ static bool guarded_alloc_initialized = []
opts.MaxSimultaneousAllocations = 1024;
if (!env_options_raw || !std::string_view{env_options_raw}.contains("SampleRate"))
opts.SampleRate = 50000;
opts.SampleRate = 10000;
const char * collect_stacktraces = std::getenv("GWP_ASAN_COLLECT_STACKTRACES"); // NOLINT(concurrency-mt-unsafe)
if (collect_stacktraces && std::string_view{collect_stacktraces} == "1")
opts.Backtrace = getBackTrace;
GuardedAlloc.init(opts);
return true;

26
src/Common/Jemalloc.cpp
View File

@ -46,6 +46,20 @@ void checkJemallocProfilingEnabled()
"set: MALLOC_CONF=background_thread:true,prof:true");
}
template <typename T>
void setJemallocValue(const char * name, T value)
{
T old_value;
size_t old_value_size = sizeof(T);
if (mallctl(name, &old_value, &old_value_size, reinterpret_cast<void*>(&value), sizeof(T)))
{
LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name);
return;
}
LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value);
}
void setJemallocProfileActive(bool value)
{
checkJemallocProfilingEnabled();
@ -58,7 +72,7 @@ void setJemallocProfileActive(bool value)
return;
}
mallctl("prof.active", nullptr, nullptr, &value, sizeof(bool));
setJemallocValue("prof.active", value);
LOG_TRACE(getLogger("SystemJemalloc"), "Profiling is {}", value ? "enabled" : "disabled");
}
@ -84,6 +98,16 @@ std::string flushJemallocProfile(const std::string & file_prefix)
return profile_dump_path;
}
void setJemallocBackgroundThreads(bool enabled)
{
setJemallocValue("background_thread", enabled);
}
void setJemallocMaxBackgroundThreads(size_t max_threads)
{
setJemallocValue("max_background_threads", max_threads);
}
}
#endif

4
src/Common/Jemalloc.h
View File

@ -17,6 +17,10 @@ void setJemallocProfileActive(bool value);
std::string flushJemallocProfile(const std::string & file_prefix);
void setJemallocBackgroundThreads(bool enabled);
void setJemallocMaxBackgroundThreads(size_t max_threads);
}
#endif

46
src/Client/QueryFuzzer.cpp → src/Common/QueryFuzzer.cpp
View File

@ -68,22 +68,21 @@ Field QueryFuzzer::getRandomField(int type)
{
case 0:
{
return bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
/ sizeof(*bad_int64_values))];
return bad_int64_values[fuzz_rand() % std::size(bad_int64_values)];
}
case 1:
{
static constexpr double values[]
= {NAN, INFINITY, -INFINITY, 0., -0., 0.0001, 0.5, 0.9999,
1., 1.0001, 2., 10.0001, 100.0001, 1000.0001, 1e10, 1e20,
FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % std::size(values)];
}
case 2:
{
static constexpr UInt64 scales[] = {0, 1, 2, 10};
return DecimalField<Decimal64>(
bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values) / sizeof(*bad_int64_values))],
static_cast<UInt32>(scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))])
bad_int64_values[fuzz_rand() % std::size(bad_int64_values)],
static_cast<UInt32>(scales[fuzz_rand() % std::size(scales)])
);
}
default:
@ -165,7 +164,8 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.erase(arr.begin() + pos);
std::cerr << "erased\n";
if (debug_stream)
*debug_stream << "erased\n";
}
if (fuzz_rand() % 5 == 0)
@ -174,12 +174,14 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
std::cerr << fmt::format("inserted (pos {})\n", pos);
if (debug_stream)
*debug_stream << fmt::format("inserted (pos {})\n", pos);
}
else
{
arr.insert(arr.begin(), getRandomField(0));
std::cerr << "inserted (0)\n";
if (debug_stream)
*debug_stream << "inserted (0)\n";
}
}
@ -197,7 +199,9 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.erase(arr.begin() + pos);
std::cerr << "erased\n";
if (debug_stream)
*debug_stream << "erased\n";
}
if (fuzz_rand() % 5 == 0)
@ -206,12 +210,16 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
std::cerr << fmt::format("inserted (pos {})\n", pos);
if (debug_stream)
*debug_stream << fmt::format("inserted (pos {})\n", pos);
}
else
{
arr.insert(arr.begin(), getRandomField(0));
std::cerr << "inserted (0)\n";
if (debug_stream)
*debug_stream << "inserted (0)\n";
}
}
@ -344,7 +352,8 @@ void QueryFuzzer::fuzzOrderByList(IAST * ast)
}
else
{
std::cerr << "No random column.\n";
if (debug_stream)
*debug_stream << "No random column.\n";
}
}
@ -378,7 +387,8 @@ void QueryFuzzer::fuzzColumnLikeExpressionList(IAST * ast)
if (col)
impl->children.insert(pos, col);
else
std::cerr << "No random column.\n";
if (debug_stream)
*debug_stream << "No random column.\n";
}
// We don't have to recurse here to fuzz the children, this is handled by
@ -1361,11 +1371,15 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast)
collectFuzzInfoMain(ast);
fuzz(ast);
std::cout << std::endl;
WriteBufferFromOStream ast_buf(std::cout, 4096);
if (out_stream)
{
*out_stream << std::endl;
WriteBufferFromOStream ast_buf(*out_stream, 4096);
formatAST(*ast, ast_buf, false /*highlight*/);
ast_buf.finalize();
std::cout << std::endl << std::endl;
*out_stream << std::endl << std::endl;
}
}
}

35
src/Client/QueryFuzzer.h → src/Common/QueryFuzzer.h
View File

@ -35,9 +35,31 @@ struct ASTWindowDefinition;
* queries, so you want to feed it a lot of queries to get some interesting mix
* of them. Normally we feed SQL regression tests to it.
*/
struct QueryFuzzer
class QueryFuzzer
{
pcg64 fuzz_rand{randomSeed()};
public:
explicit QueryFuzzer(pcg64 fuzz_rand_ = randomSeed(), std::ostream * out_stream_ = nullptr, std::ostream * debug_stream_ = nullptr)
: fuzz_rand(fuzz_rand_)
, out_stream(out_stream_)
, debug_stream(debug_stream_)
{
}
// This is the only function you have to call -- it will modify the passed
// ASTPtr to point to new AST with some random changes.
void fuzzMain(ASTPtr & ast);
ASTs getInsertQueriesForFuzzedTables(const String & full_query);
ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
void notifyQueryFailed(ASTPtr ast);
static bool isSuitableForFuzzing(const ASTCreateQuery & create);
private:
pcg64 fuzz_rand;
std::ostream * out_stream = nullptr;
std::ostream * debug_stream = nullptr;
// We add elements to expression lists with fixed probability. Some elements
// are so large, that the expected number of elements we add to them is
@ -66,10 +88,6 @@ struct QueryFuzzer
std::unordered_map<std::string, size_t> index_of_fuzzed_table;
std::set<IAST::Hash> created_tables_hashes;
// This is the only function you have to call -- it will modify the passed
// ASTPtr to point to new AST with some random changes.
void fuzzMain(ASTPtr & ast);
// Various helper functions follow, normally you shouldn't have to call them.
Field getRandomField(int type);
Field fuzzField(Field field);
@ -77,9 +95,6 @@ struct QueryFuzzer
ASTPtr getRandomExpressionList();
DataTypePtr fuzzDataType(DataTypePtr type);
DataTypePtr getRandomType();
ASTs getInsertQueriesForFuzzedTables(const String & full_query);
ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
void notifyQueryFailed(ASTPtr ast);
void replaceWithColumnLike(ASTPtr & ast);
void replaceWithTableLike(ASTPtr & ast);
void fuzzOrderByElement(ASTOrderByElement * elem);
@ -102,8 +117,6 @@ struct QueryFuzzer
void addTableLike(ASTPtr ast);
void addColumnLike(ASTPtr ast);
void collectFuzzInfoRecurse(ASTPtr ast);
static bool isSuitableForFuzzing(const ASTCreateQuery & create);
};
}

1
src/Common/config.h.in
View File

@ -63,6 +63,7 @@
#cmakedefine01 USE_BCRYPT
#cmakedefine01 USE_LIBARCHIVE
#cmakedefine01 USE_POCKETFFT
#cmakedefine01 USE_PROMETHEUS_PROTOBUFS
/// This is needed for .incbin in assembly. For some reason, include paths don't work there in presence of LTO.
/// That's why we use absolute paths.

1
src/Common/formatIPv6.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <algorithm>
#include <array>
#include <cstring>
#include <type_traits>
#include <utility>

1
src/Coordination/Changelog.h
View File

@ -5,6 +5,7 @@
#include <Common/ConcurrentBoundedQueue.h>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <future>

1
src/Coordination/FourLetterCommand.h
View File

@ -2,6 +2,7 @@
#include "config.h"
#include <memory>
#include <unordered_map>
#include <string>
#include <boost/noncopyable.hpp>

2
src/Core/ServerSettings.h
View File

@ -153,7 +153,7 @@ namespace DB
M(Bool, enable_azure_sdk_logging, false, "Enables logging from Azure sdk", 0) \
M(String, merge_workload, "default", "Name of workload to be used to access resources for all merges (may be overridden by a merge tree setting)", 0) \
M(String, mutation_workload, "default", "Name of workload to be used to access resources for all mutations (may be overridden by a merge tree setting)", 0) \
M(Double, gwp_asan_force_sample_probability, 0, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \
M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \
M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \
/// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp

5
src/Core/Settings.h
View File

@ -125,6 +125,9 @@ class IColumn;
M(Bool, s3_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in S3 table engine", 0) \
M(Bool, hdfs_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in HDFS table engine", 0) \
M(Bool, azure_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in AzureBlobStorage table engine", 0) \
M(UInt64, azure_sdk_max_retries, 10, "Maximum number of retries in azure sdk", 0) \
M(UInt64, azure_sdk_retry_initial_backoff_ms, 10, "Minimal backoff between retries in azure sdk", 0) \
M(UInt64, azure_sdk_retry_max_backoff_ms, 1000, "Maximal backoff between retries in azure sdk", 0) \
M(Bool, s3_validate_request_settings, true, "Validate S3 request settings", 0) \
M(Bool, s3_disable_checksum, S3::DEFAULT_DISABLE_CHECKSUM, "Do not calculate a checksum when sending a file to S3. This speeds up writes by avoiding excessive processing passes on a file. It is mostly safe as the data of MergeTree tables is checksummed by ClickHouse anyway, and when S3 is accessed with HTTPS, the TLS layer already provides integrity while transferring through the network. While additional checksums on S3 give defense in depth.", 0) \
M(UInt64, s3_retry_attempts, S3::DEFAULT_RETRY_ATTEMPTS, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \
@ -399,7 +402,7 @@ class IColumn;
M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \
M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \
M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \
M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", 0) \
M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", IMPORTANT) \
M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \
M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \
M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \

3
src/Core/SettingsChangesHistory.cpp
View File

@ -65,6 +65,9 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
{"database_replicated_allow_replicated_engine_arguments", 1, 0, "Don't allow explicit arguments by default"},
{"database_replicated_allow_explicit_uuid", 0, 0, "Added a new setting to disallow explicitly specifying table UUID"},
{"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
{"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
{"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
}},
{"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
{"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

4
src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
View File

@ -39,7 +39,7 @@ struct WriteBufferFromAzureBlobStorage::PartData
size_t data_size = 0;
};
BufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureObjectStorageSettings & settings)
BufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureBlobStorage::RequestSettings & settings)
{
BufferAllocationPolicy::Settings allocation_settings;
allocation_settings.strict_size = settings.strict_upload_part_size;
@ -57,7 +57,7 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
const String & blob_path_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_)
: WriteBufferFromFileBase(buf_size_, nullptr, 0)
, log(getLogger("WriteBufferFromAzureBlobStorage"))

2
src/Disks/IO/WriteBufferFromAzureBlobStorage.h
View File

@ -35,7 +35,7 @@ public:
const String & blob_path_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
~WriteBufferFromAzureBlobStorage() override;

272
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp
View File

@ -1,272 +0,0 @@
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#if USE_AZURE_BLOB_STORAGE
#include <Common/Exception.h>
#include <Common/re2.h>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context.h>
using namespace Azure::Storage::Blobs;
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
void validateStorageAccountUrl(const String & storage_account_url)
{
const auto * storage_account_url_pattern_str = R"(http(()|s)://[a-z0-9-.:]+(()|/)[a-z0-9]*(()|/))";
static const RE2 storage_account_url_pattern(storage_account_url_pattern_str);
if (!re2::RE2::FullMatch(storage_account_url, storage_account_url_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage URL is not valid, should follow the format: {}, got: {}", storage_account_url_pattern_str, storage_account_url);
}
void validateContainerName(const String & container_name)
{
auto len = container_name.length();
if (len < 3 || len > 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should have length between 3 and 64, but has length: {}", len);
const auto * container_name_pattern_str = R"([a-z][a-z0-9-]+)";
static const RE2 container_name_pattern(container_name_pattern_str);
if (!re2::RE2::FullMatch(container_name, container_name_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should follow the format: {}, got: {}",
container_name_pattern_str, container_name);
}
AzureBlobStorageEndpoint processAzureBlobStorageEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
String storage_url;
String account_name;
String container_name;
String prefix;
if (config.has(config_prefix + ".endpoint"))
{
String endpoint = config.getString(config_prefix + ".endpoint");
/// For some authentication methods account name is not present in the endpoint
/// 'endpoint_contains_account_name' bool is used to understand how to split the endpoint (default : true)
bool endpoint_contains_account_name = config.getBool(config_prefix + ".endpoint_contains_account_name", true);
size_t pos = endpoint.find("//");
if (pos == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected '//' in endpoint");
if (endpoint_contains_account_name)
{
size_t acc_pos_begin = endpoint.find('/', pos+2);
if (acc_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected account_name in endpoint");
storage_url = endpoint.substr(0,acc_pos_begin);
size_t acc_pos_end = endpoint.find('/',acc_pos_begin+1);
if (acc_pos_end == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
account_name = endpoint.substr(acc_pos_begin+1,(acc_pos_end-acc_pos_begin)-1);
size_t cont_pos_end = endpoint.find('/', acc_pos_end+1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(acc_pos_end+1,(cont_pos_end-acc_pos_end)-1);
prefix = endpoint.substr(cont_pos_end+1);
}
else
{
container_name = endpoint.substr(acc_pos_end+1);
}
}
else
{
size_t cont_pos_begin = endpoint.find('/', pos+2);
if (cont_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
storage_url = endpoint.substr(0,cont_pos_begin);
size_t cont_pos_end = endpoint.find('/',cont_pos_begin+1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(cont_pos_begin+1,(cont_pos_end-cont_pos_begin)-1);
prefix = endpoint.substr(cont_pos_end+1);
}
else
{
container_name = endpoint.substr(cont_pos_begin+1);
}
}
}
else if (config.has(config_prefix + ".connection_string"))
{
storage_url = config.getString(config_prefix + ".connection_string");
container_name = config.getString(config_prefix + ".container_name");
}
else if (config.has(config_prefix + ".storage_account_url"))
{
storage_url = config.getString(config_prefix + ".storage_account_url");
validateStorageAccountUrl(storage_url);
container_name = config.getString(config_prefix + ".container_name");
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `storage_account_url` or `connection_string` or `endpoint` in config");
if (!container_name.empty())
validateContainerName(container_name);
std::optional<bool> container_already_exists {};
if (config.has(config_prefix + ".container_already_exists"))
container_already_exists = {config.getBool(config_prefix + ".container_already_exists")};
return {storage_url, account_name, container_name, prefix, container_already_exists};
}
template <class T>
std::unique_ptr<T> getClientWithConnectionString(const String & connection_str, const String & container_name, const BlobClientOptions & client_options) = delete;
template<>
std::unique_ptr<BlobServiceClient> getClientWithConnectionString(const String & connection_str, const String & /*container_name*/, const BlobClientOptions & client_options)
{
return std::make_unique<BlobServiceClient>(BlobServiceClient::CreateFromConnectionString(connection_str, client_options));
}
template<>
std::unique_ptr<BlobContainerClient> getClientWithConnectionString(const String & connection_str, const String & container_name, const BlobClientOptions & client_options)
{
return std::make_unique<BlobContainerClient>(BlobContainerClient::CreateFromConnectionString(connection_str, container_name, client_options));
}
template <class T>
std::unique_ptr<T> getAzureBlobStorageClientWithAuth(
const String & url,
const String & container_name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
const Azure::Storage::Blobs::BlobClientOptions & client_options)
{
std::string connection_str;
if (config.has(config_prefix + ".connection_string"))
connection_str = config.getString(config_prefix + ".connection_string");
if (!connection_str.empty())
return getClientWithConnectionString<T>(connection_str, container_name, client_options);
if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name"))
{
auto storage_shared_key_credential = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(
config.getString(config_prefix + ".account_name"),
config.getString(config_prefix + ".account_key")
);
return std::make_unique<T>(url, storage_shared_key_credential, client_options);
}
if (config.getBool(config_prefix + ".use_workload_identity", false))
{
auto workload_identity_credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
return std::make_unique<T>(url, workload_identity_credential, client_options);
}
auto managed_identity_credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
return std::make_unique<T>(url, managed_identity_credential, client_options);
}
Azure::Storage::Blobs::BlobClientOptions getAzureBlobClientOptions(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
Azure::Core::Http::Policies::RetryOptions retry_options;
retry_options.MaxRetries = config.getUInt(config_prefix + ".max_tries", 10);
retry_options.RetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_initial_backoff_ms", 10));
retry_options.MaxRetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_max_backoff_ms", 1000));
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
CurlOptions curl_options;
curl_options.NoSignal = true;
if (config.has(config_prefix + ".curl_ip_resolve"))
{
auto value = config.getString(config_prefix + ".curl_ip_resolve");
if (value == "ipv4")
curl_options.IPResolve = CurlOptions::CURL_IPRESOLVE_V4;
else if (value == "ipv6")
curl_options.IPResolve = CurlOptions::CURL_IPRESOLVE_V6;
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected value for option 'curl_ip_resolve': {}. Expected one of 'ipv4' or 'ipv6'", value);
}
Azure::Storage::Blobs::BlobClientOptions client_options;
client_options.Retry = retry_options;
client_options.Transport.Transport = std::make_shared<Azure::Core::Http::CurlTransport>(curl_options);
client_options.ClickhouseOptions = Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true};
return client_options;
}
std::unique_ptr<BlobContainerClient> getAzureBlobContainerClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
auto endpoint = processAzureBlobStorageEndpoint(config, config_prefix);
auto container_name = endpoint.container_name;
auto final_url = endpoint.getEndpoint();
auto client_options = getAzureBlobClientOptions(config, config_prefix);
if (endpoint.container_already_exists.value_or(false))
return getAzureBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix, client_options);
auto blob_service_client = getAzureBlobStorageClientWithAuth<BlobServiceClient>(endpoint.getEndpointWithoutContainer(), container_name, config, config_prefix, client_options);
try
{
return std::make_unique<BlobContainerClient>(blob_service_client->CreateBlobContainer(container_name).Value);
}
catch (const Azure::Storage::StorageException & e)
{
/// If container_already_exists is not set (in config), ignore already exists error.
/// (Conflict - The specified container already exists)
if (!endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
return getAzureBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix, client_options);
throw;
}
}
std::unique_ptr<AzureObjectStorageSettings> getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context)
{
std::unique_ptr<AzureObjectStorageSettings> settings = std::make_unique<AzureObjectStorageSettings>();
settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", context->getSettings().azure_max_single_part_upload_size);
settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024);
settings->max_single_read_retries = config.getInt(config_prefix + ".max_single_read_retries", 3);
settings->max_single_download_retries = config.getInt(config_prefix + ".max_single_download_retries", 3);
settings->list_object_keys_size = config.getInt(config_prefix + ".list_object_keys_size", 1000);
settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", context->getSettings().azure_min_upload_part_size);
settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", context->getSettings().azure_max_upload_part_size);
settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size);
settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false);
settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", 50000);
settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries);
settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", context->getSettings().azure_max_inflight_parts_for_one_file);
settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", context->getSettings().azure_strict_upload_part_size);
settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", context->getSettings().azure_upload_part_size_multiply_factor);
settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", context->getSettings().azure_upload_part_size_multiply_parts_count_threshold);
return settings;
}
}
#endif

58
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h
View File

@ -1,58 +0,0 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <azure/storage/blobs.hpp>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
namespace DB
{
struct AzureBlobStorageEndpoint
{
const String storage_account_url;
const String account_name;
const String container_name;
const String prefix;
const std::optional<bool> container_already_exists;
String getEndpoint()
{
String url = storage_account_url;
if (url.ends_with('/'))
url.pop_back();
if (!account_name.empty())
url += "/" + account_name;
if (!container_name.empty())
url += "/" + container_name;
if (!prefix.empty())
url += "/" + prefix;
return url;
}
String getEndpointWithoutContainer()
{
String url = storage_account_url;
if (!account_name.empty())
url += "/" + account_name;
return url;
}
};
std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient> getAzureBlobContainerClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
AzureBlobStorageEndpoint processAzureBlobStorageEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
std::unique_ptr<AzureObjectStorageSettings> getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
}
#endif

351
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp Normal file
View File

@ -0,0 +1,351 @@
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#if USE_AZURE_BLOB_STORAGE
#include <Common/Exception.h>
#include <Common/re2.h>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
namespace AzureBlobStorage
{
static void validateStorageAccountUrl(const String & storage_account_url)
{
const auto * storage_account_url_pattern_str = R"(http(()|s)://[a-z0-9-.:]+(()|/)[a-z0-9]*(()|/))";
static const RE2 storage_account_url_pattern(storage_account_url_pattern_str);
if (!re2::RE2::FullMatch(storage_account_url, storage_account_url_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage URL is not valid, should follow the format: {}, got: {}", storage_account_url_pattern_str, storage_account_url);
}
static void validateContainerName(const String & container_name)
{
auto len = container_name.length();
if (len < 3 || len > 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should have length between 3 and 64, but has length: {}", len);
const auto * container_name_pattern_str = R"([a-z][a-z0-9-]+)";
static const RE2 container_name_pattern(container_name_pattern_str);
if (!re2::RE2::FullMatch(container_name, container_name_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should follow the format: {}, got: {}",
container_name_pattern_str, container_name);
}
static bool isConnectionString(const std::string & candidate)
{
return !candidate.starts_with("http");
}
String ConnectionParams::getConnectionURL() const
{
if (std::holds_alternative<ConnectionString>(auth_method))
{
auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url);
return parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl();
}
return endpoint.storage_account_url;
}
std::unique_ptr<ServiceClient> ConnectionParams::createForService() const
{
return std::visit([this]<typename T>(const T & auth)
{
if constexpr (std::is_same_v<T, ConnectionString>)
return std::make_unique<ServiceClient>(ServiceClient::CreateFromConnectionString(auth.toUnderType(), client_options));
else
return std::make_unique<ServiceClient>(endpoint.getEndpointWithoutContainer(), auth, client_options);
}, auth_method);
}
std::unique_ptr<ContainerClient> ConnectionParams::createForContainer() const
{
return std::visit([this]<typename T>(const T & auth)
{
if constexpr (std::is_same_v<T, ConnectionString>)
return std::make_unique<ContainerClient>(ContainerClient::CreateFromConnectionString(auth.toUnderType(), endpoint.container_name, client_options));
else
return std::make_unique<ContainerClient>(endpoint.getEndpoint(), auth, client_options);
}, auth_method);
}
Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
String storage_url;
String account_name;
String container_name;
String prefix;
auto get_container_name = [&]
{
if (config.has(config_prefix + ".container_name"))
return config.getString(config_prefix + ".container_name");
if (config.has(config_prefix + ".container"))
return config.getString(config_prefix + ".container");
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `container` or `container_name` parameter in config");
};
if (config.has(config_prefix + ".endpoint"))
{
String endpoint = config.getString(config_prefix + ".endpoint");
/// For some authentication methods account name is not present in the endpoint
/// 'endpoint_contains_account_name' bool is used to understand how to split the endpoint (default : true)
bool endpoint_contains_account_name = config.getBool(config_prefix + ".endpoint_contains_account_name", true);
size_t pos = endpoint.find("//");
if (pos == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected '//' in endpoint");
if (endpoint_contains_account_name)
{
size_t acc_pos_begin = endpoint.find('/', pos + 2);
if (acc_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected account_name in endpoint");
storage_url = endpoint.substr(0, acc_pos_begin);
size_t acc_pos_end = endpoint.find('/', acc_pos_begin + 1);
if (acc_pos_end == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
account_name = endpoint.substr(acc_pos_begin + 1, acc_pos_end - acc_pos_begin - 1);
size_t cont_pos_end = endpoint.find('/', acc_pos_end + 1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(acc_pos_end + 1, cont_pos_end - acc_pos_end - 1);
prefix = endpoint.substr(cont_pos_end + 1);
}
else
{
container_name = endpoint.substr(acc_pos_end + 1);
}
}
else
{
size_t cont_pos_begin = endpoint.find('/', pos + 2);
if (cont_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
storage_url = endpoint.substr(0, cont_pos_begin);
size_t cont_pos_end = endpoint.find('/', cont_pos_begin + 1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(cont_pos_begin + 1,cont_pos_end - cont_pos_begin - 1);
prefix = endpoint.substr(cont_pos_end + 1);
}
else
{
container_name = endpoint.substr(cont_pos_begin + 1);
}
}
}
else if (config.has(config_prefix + ".connection_string"))
{
storage_url = config.getString(config_prefix + ".connection_string");
container_name = get_container_name();
}
else if (config.has(config_prefix + ".storage_account_url"))
{
storage_url = config.getString(config_prefix + ".storage_account_url");
validateStorageAccountUrl(storage_url);
container_name = get_container_name();
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `storage_account_url` or `connection_string` or `endpoint` in config");
if (!container_name.empty())
validateContainerName(container_name);
std::optional<bool> container_already_exists {};
if (config.has(config_prefix + ".container_already_exists"))
container_already_exists = {config.getBool(config_prefix + ".container_already_exists")};
return {storage_url, account_name, container_name, prefix, "", container_already_exists};
}
void processURL(const String & url, const String & container_name, Endpoint & endpoint, AuthMethod & auth_method)
{
endpoint.container_name = container_name;
if (isConnectionString(url))
{
endpoint.storage_account_url = url;
auth_method = ConnectionString{url};
return;
}
auto pos = url.find('?');
/// If conneciton_url does not have '?', then its not SAS
if (pos == std::string::npos)
{
endpoint.storage_account_url = url;
auth_method = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
}
else
{
endpoint.storage_account_url = url.substr(0, pos);
endpoint.sas_auth = url.substr(pos + 1);
auth_method = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
}
}
std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & params, bool readonly)
{
if (params.endpoint.container_already_exists.value_or(false) || readonly)
return params.createForContainer();
try
{
auto service_client = params.createForService();
return std::make_unique<ContainerClient>(service_client->CreateBlobContainer(params.endpoint.container_name).Value);
}
catch (const Azure::Storage::StorageException & e)
{
/// If container_already_exists is not set (in config), ignore already exists error.
/// (Conflict - The specified container already exists)
if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
return params.createForContainer();
throw;
}
}
AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name"))
{
return std::make_shared<Azure::Storage::StorageSharedKeyCredential>(
config.getString(config_prefix + ".account_name"),
config.getString(config_prefix + ".account_key")
);
}
if (config.has(config_prefix + ".connection_string"))
return ConnectionString{config.getString(config_prefix + ".connection_string")};
if (config.getBool(config_prefix + ".use_workload_identity", false))
return std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
return std::make_shared<Azure::Identity::ManagedIdentityCredential>();
}
BlobClientOptions getClientOptions(const RequestSettings & settings, bool for_disk)
{
Azure::Core::Http::Policies::RetryOptions retry_options;
retry_options.MaxRetries = static_cast<Int32>(settings.sdk_max_retries);
retry_options.RetryDelay = std::chrono::milliseconds(settings.sdk_retry_initial_backoff_ms);
retry_options.MaxRetryDelay = std::chrono::milliseconds(settings.sdk_retry_max_backoff_ms);
Azure::Core::Http::CurlTransportOptions curl_options;
curl_options.NoSignal = true;
curl_options.IPResolve = settings.curl_ip_resolve;
Azure::Storage::Blobs::BlobClientOptions client_options;
client_options.Retry = retry_options;
client_options.Transport.Transport = std::make_shared<Azure::Core::Http::CurlTransport>(curl_options);
client_options.ClickhouseOptions = Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=for_disk};
return client_options;
}
std::unique_ptr<RequestSettings> getRequestSettings(const Settings & query_settings)
{
auto settings = std::make_unique<RequestSettings>();
settings->max_single_part_upload_size = query_settings.azure_max_single_part_upload_size;
settings->max_single_read_retries = query_settings.azure_max_single_read_retries;
settings->max_single_download_retries = query_settings.azure_max_single_read_retries;
settings->list_object_keys_size = query_settings.azure_list_object_keys_size;
settings->min_upload_part_size = query_settings.azure_min_upload_part_size;
settings->max_upload_part_size = query_settings.azure_max_upload_part_size;
settings->max_single_part_copy_size = query_settings.azure_max_single_part_copy_size;
settings->max_blocks_in_multipart_upload = query_settings.azure_max_blocks_in_multipart_upload;
settings->max_unexpected_write_error_retries = query_settings.azure_max_unexpected_write_error_retries;
settings->max_inflight_parts_for_one_file = query_settings.azure_max_inflight_parts_for_one_file;
settings->strict_upload_part_size = query_settings.azure_strict_upload_part_size;
settings->upload_part_size_multiply_factor = query_settings.azure_upload_part_size_multiply_factor;
settings->upload_part_size_multiply_parts_count_threshold = query_settings.azure_upload_part_size_multiply_parts_count_threshold;
settings->sdk_max_retries = query_settings.azure_sdk_max_retries;
settings->sdk_retry_initial_backoff_ms = query_settings.azure_sdk_retry_initial_backoff_ms;
settings->sdk_retry_max_backoff_ms = query_settings.azure_sdk_retry_max_backoff_ms;
return settings;
}
std::unique_ptr<RequestSettings> getRequestSettingsForBackup(const Settings & query_settings, bool use_native_copy)
{
auto settings = getRequestSettings(query_settings);
settings->use_native_copy = use_native_copy;
return settings;
}
std::unique_ptr<RequestSettings> getRequestSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context)
{
auto settings = std::make_unique<RequestSettings>();
const auto & settings_ref = context->getSettingsRef();
settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024);
settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false);
settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", settings_ref.azure_max_single_part_upload_size);
settings->max_single_read_retries = config.getUInt64(config_prefix + ".max_single_read_retries", settings_ref.azure_max_single_read_retries);
settings->max_single_download_retries = config.getUInt64(config_prefix + ".max_single_download_retries", settings_ref.azure_max_single_read_retries);
settings->list_object_keys_size = config.getUInt64(config_prefix + ".list_object_keys_size", settings_ref.azure_list_object_keys_size);
settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", settings_ref.azure_min_upload_part_size);
settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", settings_ref.azure_max_upload_part_size);
settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", settings_ref.azure_max_single_part_copy_size);
settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", settings_ref.azure_max_blocks_in_multipart_upload);
settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", settings_ref.azure_max_unexpected_write_error_retries);
settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", settings_ref.azure_max_inflight_parts_for_one_file);
settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", settings_ref.azure_strict_upload_part_size);
settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", settings_ref.azure_upload_part_size_multiply_factor);
settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", settings_ref.azure_upload_part_size_multiply_parts_count_threshold);
settings->sdk_max_retries = config.getUInt64(config_prefix + ".max_tries", settings_ref.azure_sdk_max_retries);
settings->sdk_retry_initial_backoff_ms = config.getUInt64(config_prefix + ".retry_initial_backoff_ms", settings_ref.azure_sdk_retry_initial_backoff_ms);
settings->sdk_retry_max_backoff_ms = config.getUInt64(config_prefix + ".retry_max_backoff_ms", settings_ref.azure_sdk_retry_max_backoff_ms);
if (config.has(config_prefix + ".curl_ip_resolve"))
{
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
auto value = config.getString(config_prefix + ".curl_ip_resolve");
if (value == "ipv4")
settings->curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_V4;
else if (value == "ipv6")
settings->curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_V6;
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected value for option 'curl_ip_resolve': {}. Expected one of 'ipv4' or 'ipv6'", value);
}
return settings;
}
}
}
#endif

138
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h Normal file
View File

@ -0,0 +1,138 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <azure/storage/blobs.hpp>
#include <azure/storage/blobs/blob_client.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <azure/storage/blobs/blob_service_client.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <Core/Settings.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context_fwd.h>
#include <base/strong_typedef.h>
namespace DB
{
namespace AzureBlobStorage
{
using ServiceClient = Azure::Storage::Blobs::BlobServiceClient;
using ContainerClient = Azure::Storage::Blobs::BlobContainerClient;
using BlobClient = Azure::Storage::Blobs::BlobClient;
using BlobClientOptions = Azure::Storage::Blobs::BlobClientOptions;
struct RequestSettings
{
RequestSettings() = default;
size_t max_single_part_upload_size = 100 * 1024 * 1024; /// NOTE: on 32-bit machines it will be at most 4GB, but size_t is also used in BufferBase for offset
size_t min_bytes_for_seek = 1024 * 1024;
size_t max_single_read_retries = 3;
size_t max_single_download_retries = 3;
size_t list_object_keys_size = 1000;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t max_single_part_copy_size = 256 * 1024 * 1024;
size_t max_unexpected_write_error_retries = 4;
size_t max_inflight_parts_for_one_file = 20;
size_t max_blocks_in_multipart_upload = 50000;
size_t strict_upload_part_size = 0;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
size_t sdk_max_retries = 10;
size_t sdk_retry_initial_backoff_ms = 10;
size_t sdk_retry_max_backoff_ms = 1000;
bool use_native_copy = false;
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
CurlOptions::CurlOptIPResolve curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_WHATEVER;
};
struct Endpoint
{
String storage_account_url;
String account_name;
String container_name;
String prefix;
String sas_auth;
std::optional<bool> container_already_exists;
String getEndpoint() const
{
String url = storage_account_url;
if (url.ends_with('/'))
url.pop_back();
if (!account_name.empty())
url += "/" + account_name;
if (!container_name.empty())
url += "/" + container_name;
if (!prefix.empty())
url += "/" + prefix;
if (!sas_auth.empty())
url += "?" + sas_auth;
return url;
}
String getEndpointWithoutContainer() const
{
String url = storage_account_url;
if (!account_name.empty())
url += "/" + account_name;
if (!sas_auth.empty())
url += "?" + sas_auth;
return url;
}
};
using ConnectionString = StrongTypedef<String, struct ConnectionStringTag>;
using AuthMethod = std::variant<
ConnectionString,
std::shared_ptr<Azure::Storage::StorageSharedKeyCredential>,
std::shared_ptr<Azure::Identity::WorkloadIdentityCredential>,
std::shared_ptr<Azure::Identity::ManagedIdentityCredential>>;
struct ConnectionParams
{
Endpoint endpoint;
AuthMethod auth_method;
BlobClientOptions client_options;
String getContainer() const { return endpoint.container_name; }
String getConnectionURL() const;
std::unique_ptr<ServiceClient> createForService() const;
std::unique_ptr<ContainerClient> createForContainer() const;
};
Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
void processURL(const String & url, const String & container_name, Endpoint & endpoint, AuthMethod & auth_method);
std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & params, bool readonly);
BlobClientOptions getClientOptions(const RequestSettings & settings, bool for_disk);
AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
std::unique_ptr<RequestSettings> getRequestSettings(const Settings & query_settings);
std::unique_ptr<RequestSettings> getRequestSettingsForBackup(const Settings & query_settings, bool use_native_copy);
std::unique_ptr<RequestSettings> getRequestSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
}
}
#endif

53
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
View File

@ -9,7 +9,7 @@
#include <Disks/IO/ReadBufferFromRemoteFSGather.h>
#include <Disks/IO/AsynchronousBoundedReadBuffer.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Disks/ObjectStorages/ObjectStorageIteratorAsync.h>
#include <Interpreters/Context.h>
#include <Common/logger_useful.h>
@ -104,7 +104,7 @@ private:
AzureObjectStorage::AzureObjectStorage(
const String & name_,
AzureClientPtr && client_,
ClientPtr && client_,
SettingsPtr && settings_,
const String & object_namespace_,
const String & description_)
@ -397,24 +397,49 @@ void AzureObjectStorage::copyObject( /// NOLINT
}
void AzureObjectStorage::applyNewSettings(
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
ContextPtr context, const ApplyNewSettingsOptions &)
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
ContextPtr context,
const ApplyNewSettingsOptions & options)
{
auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
auto new_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
settings.set(std::move(new_settings));
/// We don't update client
if (!options.allow_client_change)
return;
bool is_client_for_disk = client.get()->GetClickhouseOptions().IsClientForDisk;
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*settings.get(), is_client_for_disk),
};
auto new_client = AzureBlobStorage::getContainerClient(params, /*readonly=*/ true);
client.set(std::move(new_client));
}
std::unique_ptr<IObjectStorage> AzureObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
std::unique_ptr<IObjectStorage> AzureObjectStorage::cloneObjectStorage(
const std::string & new_namespace,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
ContextPtr context)
{
return std::make_unique<AzureObjectStorage>(
name,
getAzureBlobContainerClient(config, config_prefix),
getAzureBlobStorageSettings(config, config_prefix, context),
object_namespace,
description
);
auto new_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
bool is_client_for_disk = client.get()->GetClickhouseOptions().IsClientForDisk;
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*new_settings, is_client_for_disk),
};
auto new_client = AzureBlobStorage::getContainerClient(params, /*readonly=*/ true);
return std::make_unique<AzureObjectStorage>(name, std::move(new_client), std::move(new_settings), new_namespace, params.endpoint.getEndpointWithoutContainer());
}
}

76
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
View File

@ -7,6 +7,8 @@
#include <Disks/ObjectStorages/IObjectStorage.h>
#include <Common/MultiVersion.h>
#include <azure/storage/blobs.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
namespace Poco
{
@ -16,71 +18,15 @@ class Logger;
namespace DB
{
struct AzureObjectStorageSettings
{
AzureObjectStorageSettings(
uint64_t max_single_part_upload_size_,
uint64_t min_bytes_for_seek_,
int max_single_read_retries_,
int max_single_download_retries_,
int list_object_keys_size_,
size_t min_upload_part_size_,
size_t max_upload_part_size_,
size_t max_single_part_copy_size_,
bool use_native_copy_,
size_t max_unexpected_write_error_retries_,
size_t max_inflight_parts_for_one_file_,
size_t strict_upload_part_size_,
size_t upload_part_size_multiply_factor_,
size_t upload_part_size_multiply_parts_count_threshold_)
: max_single_part_upload_size(max_single_part_upload_size_)
, min_bytes_for_seek(min_bytes_for_seek_)
, max_single_read_retries(max_single_read_retries_)
, max_single_download_retries(max_single_download_retries_)
, list_object_keys_size(list_object_keys_size_)
, min_upload_part_size(min_upload_part_size_)
, max_upload_part_size(max_upload_part_size_)
, max_single_part_copy_size(max_single_part_copy_size_)
, use_native_copy(use_native_copy_)
, max_unexpected_write_error_retries(max_unexpected_write_error_retries_)
, max_inflight_parts_for_one_file(max_inflight_parts_for_one_file_)
, strict_upload_part_size(strict_upload_part_size_)
, upload_part_size_multiply_factor(upload_part_size_multiply_factor_)
, upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_)
{
}
AzureObjectStorageSettings() = default;
size_t max_single_part_upload_size = 100 * 1024 * 1024; /// NOTE: on 32-bit machines it will be at most 4GB, but size_t is also used in BufferBase for offset
uint64_t min_bytes_for_seek = 1024 * 1024;
size_t max_single_read_retries = 3;
size_t max_single_download_retries = 3;
int list_object_keys_size = 1000;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t max_single_part_copy_size = 256 * 1024 * 1024;
bool use_native_copy = false;
size_t max_unexpected_write_error_retries = 4;
size_t max_inflight_parts_for_one_file = 20;
size_t max_blocks_in_multipart_upload = 50000;
size_t strict_upload_part_size = 0;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
};
using AzureClient = Azure::Storage::Blobs::BlobContainerClient;
using AzureClientPtr = std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient>;
class AzureObjectStorage : public IObjectStorage
{
public:
using SettingsPtr = std::unique_ptr<AzureObjectStorageSettings>;
using ClientPtr = std::unique_ptr<AzureBlobStorage::ContainerClient>;
using SettingsPtr = std::unique_ptr<AzureBlobStorage::RequestSettings>;
AzureObjectStorage(
const String & name_,
AzureClientPtr && client_,
ClientPtr && client_,
SettingsPtr && settings_,
const String & object_namespace_,
const String & description_);
@ -159,12 +105,8 @@ public:
bool isRemote() const override { return true; }
std::shared_ptr<const AzureObjectStorageSettings> getSettings() { return settings.get(); }
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() override
{
return client.get();
}
std::shared_ptr<const AzureBlobStorage::RequestSettings> getSettings() const { return settings.get(); }
std::shared_ptr<const AzureBlobStorage::ContainerClient> getAzureBlobStorageClient() const override { return client.get(); }
bool supportParallelWrite() const override { return true; }
@ -174,8 +116,8 @@ private:
const String name;
/// client used to access the files in the Blob Storage cloud
MultiVersion<Azure::Storage::Blobs::BlobContainerClient> client;
MultiVersion<AzureObjectStorageSettings> settings;
MultiVersion<AzureBlobStorage::ContainerClient> client;
MultiVersion<AzureBlobStorage::RequestSettings> settings;
const String object_namespace; /// container + prefix
/// We use source url without container and prefix as description, because in Azure there are no limitations for operations between different containers.

2
src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
View File

@ -127,7 +127,7 @@ public:
const FileCacheSettings & getCacheSettings() const { return cache_settings; }
#if USE_AZURE_BLOB_STORAGE
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() override
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() const override
{
return object_storage->getAzureBlobStorageClient();
}

2
src/Disks/ObjectStorages/IObjectStorage.h
View File

@ -261,7 +261,7 @@ public:
virtual void setKeysGenerator(ObjectStorageKeysGeneratorPtr) { }
#if USE_AZURE_BLOB_STORAGE
virtual std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient()
virtual std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "This function is only implemented for AzureBlobStorage");
}

1
src/Disks/ObjectStorages/MetadataOperationsHolder.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <mutex>
#include <vector>
#include <Disks/ObjectStorages/IMetadataOperation.h>
#include <Disks/ObjectStorages/MetadataStorageTransactionState.h>
#include <Common/SharedMutex.h>

19
src/Disks/ObjectStorages/ObjectStorageFactory.cpp
View File

@ -13,7 +13,7 @@
#endif
#if USE_AZURE_BLOB_STORAGE
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#endif
#include <Disks/ObjectStorages/Web/WebObjectStorage.h>
#include <Disks/ObjectStorages/Local/LocalObjectStorage.h>
@ -317,15 +317,22 @@ void registerAzureObjectStorage(ObjectStorageFactory & factory)
const ContextPtr & context,
bool /* skip_access_check */) -> ObjectStoragePtr
{
AzureBlobStorageEndpoint endpoint = processAzureBlobStorageEndpoint(config, config_prefix);
auto azure_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*azure_settings, /*for_disk=*/ true),
};
return createObjectStorage<AzureObjectStorage>(
ObjectStorageType::Azure, config, config_prefix, name,
getAzureBlobContainerClient(config, config_prefix),
getAzureBlobStorageSettings(config, config_prefix, context),
endpoint.prefix.empty() ? endpoint.container_name : endpoint.container_name + "/" + endpoint.prefix,
endpoint.getEndpointWithoutContainer());
AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings),
params.endpoint.prefix.empty() ? params.endpoint.container_name : params.endpoint.container_name + "/" + params.endpoint.prefix,
params.endpoint.getEndpointWithoutContainer());
};
factory.registerObjectStorageType("azure_blob_storage", creator);
factory.registerObjectStorageType("azure", creator);
}

20
src/Functions/bitShiftLeft.cpp
View File

@ -5,6 +5,7 @@ namespace DB
{
namespace ErrorCodes
{
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
}
@ -24,6 +25,8 @@ struct BitShiftLeftImpl
{
if constexpr (is_big_int_v<B>)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftLeft is not implemented for big integers as second argument");
else if (b < 0 || static_cast<UInt256>(b) > 8 * sizeof(A))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) << static_cast<UInt32>(b);
else
@ -37,9 +40,12 @@ struct BitShiftLeftImpl
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftLeft is not implemented for big integers as second argument");
else
{
UInt8 word_size = 8;
/// To prevent overflow
if (static_cast<double>(b) >= (static_cast<double>(end - pos) * word_size) || b < 0)
const UInt8 word_size = 8 * sizeof(*pos);
size_t n = end - pos;
const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if (b == bit_limit)
{
// insert default value
out_vec.push_back(0);
@ -102,10 +108,12 @@ struct BitShiftLeftImpl
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftLeft is not implemented for big integers as second argument");
else
{
UInt8 word_size = 8;
const UInt8 word_size = 8;
size_t n = end - pos;
/// To prevent overflow
if (static_cast<double>(b) >= (static_cast<double>(n) * word_size) || b < 0)
const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if (b == bit_limit)
{
// insert default value
out_vec.resize_fill(out_vec.size() + n);

20
src/Functions/bitShiftRight.cpp
View File

@ -6,6 +6,7 @@ namespace DB
{
namespace ErrorCodes
{
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
}
@ -25,6 +26,8 @@ struct BitShiftRightImpl
{
if constexpr (is_big_int_v<B>)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftRight is not implemented for big integers as second argument");
else if (b < 0 || static_cast<UInt256>(b) > 8 * sizeof(A))
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) >> static_cast<UInt32>(b);
else
@ -53,9 +56,12 @@ struct BitShiftRightImpl
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftRight is not implemented for big integers as second argument");
else
{
UInt8 word_size = 8;
/// To prevent overflow
if (static_cast<double>(b) >= (static_cast<double>(end - pos) * word_size) || b < 0)
const UInt8 word_size = 8;
size_t n = end - pos;
const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if (b == bit_limit)
{
/// insert default value
out_vec.push_back(0);
@ -90,10 +96,12 @@ struct BitShiftRightImpl
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BitShiftRight is not implemented for big integers as second argument");
else
{
UInt8 word_size = 8;
const UInt8 word_size = 8;
size_t n = end - pos;
/// To prevent overflow
if (static_cast<double>(b) >= (static_cast<double>(n) * word_size) || b < 0)
const UInt128 bit_limit = static_cast<UInt128>(word_size) * n;
if (b < 0 || static_cast<decltype(bit_limit)>(b) > bit_limit)
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The number of shift positions needs to be a non-negative value and less or equal to the bit width of the value to shift");
else if (b == bit_limit)
{
// insert default value
out_vec.resize_fill(out_vec.size() + n);

2
src/IO/Archives/hasRegisteredArchiveFileExtension.cpp
View File

@ -1,5 +1,7 @@
#include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <algorithm>
#include <array>
namespace DB
{

8
src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp
View File

@ -47,7 +47,7 @@ namespace
size_t total_size_,
const String & dest_container_for_logging_,
const String & dest_blob_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_,
const Poco::Logger * log_)
: create_read_buffer(create_read_buffer_)
@ -72,7 +72,7 @@ namespace
size_t total_size;
const String & dest_container_for_logging;
const String & dest_blob;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
ThreadPoolCallbackRunnerUnsafe<void> schedule;
const Poco::Logger * log;
size_t max_single_part_upload_size;
@ -318,7 +318,7 @@ void copyDataToAzureBlobStorageFile(
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> dest_client,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule)
{
UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container_for_logging, dest_blob, settings, schedule, &Poco::Logger::get("copyDataToAzureBlobStorageFile")};
@ -335,7 +335,7 @@ void copyAzureBlobStorageFile(
size_t size,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
const ReadSettings & read_settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule)
{

4
src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h
View File

@ -28,7 +28,7 @@ void copyAzureBlobStorageFile(
size_t src_size,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
const ReadSettings & read_settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
@ -45,7 +45,7 @@ void copyDataToAzureBlobStorageFile(
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
}

15
src/IO/BufferWithOwnMemory.h
View File

@ -4,12 +4,15 @@
#include <Common/ProfileEvents.h>
#include <Common/Allocator.h>
#include <Common/GWPAsan.h>
#include <Common/Exception.h>
#include <Core/Defines.h>
#include <base/arithmeticOverflow.h>
#include "config.h"
namespace ProfileEvents
{
@ -41,10 +44,13 @@ struct Memory : boost::noncopyable, Allocator
char * m_data = nullptr;
size_t alignment = 0;
[[maybe_unused]] bool allow_gwp_asan_force_sample;
Memory() = default;
/// If alignment != 0, then allocate memory aligned to specified value.
explicit Memory(size_t size_, size_t alignment_ = 0) : alignment(alignment_)
explicit Memory(size_t size_, size_t alignment_ = 0, bool allow_gwp_asan_force_sample_ = false)
: alignment(alignment_), allow_gwp_asan_force_sample(allow_gwp_asan_force_sample_)
{
alloc(size_);
}
@ -127,6 +133,11 @@ private:
ProfileEvents::increment(ProfileEvents::IOBufferAllocs);
ProfileEvents::increment(ProfileEvents::IOBufferAllocBytes, new_capacity);
#if USE_GWP_ASAN
if (unlikely(allow_gwp_asan_force_sample && GWPAsan::shouldForceSample()))
gwp_asan::getThreadLocals()->NextSampleCounter = 1;
#endif
m_data = static_cast<char *>(Allocator::alloc(new_capacity, alignment));
m_capacity = new_capacity;
m_size = new_size;
@ -154,7 +165,7 @@ protected:
public:
/// If non-nullptr 'existing_memory' is passed, then buffer will not create its own memory and will use existing_memory without ownership.
explicit BufferWithOwnMemory(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0)
: Base(nullptr, 0), memory(existing_memory ? 0 : size, alignment)
: Base(nullptr, 0), memory(existing_memory ? 0 : size, alignment, /*allow_gwp_asan_force_sample_=*/true)
{
Base::set(existing_memory ? existing_memory : memory.data(), size);
Base::padded = !existing_memory;

30
src/Interpreters/getColumnFromBlock.cpp
View File

@ -31,6 +31,36 @@ ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & req
return castColumn({elem_column, elem_type, ""}, requested_column.type);
}
ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requested_column_type, const NameAndTypePair & requested_subcolumn)
{
const auto * elem = block.findByName(requested_subcolumn.getNameInStorage());
if (!elem)
return nullptr;
auto subcolumn_name = requested_subcolumn.getSubcolumnName();
/// If requested subcolumn is dynamic, we should first perform cast and then
/// extract the subcolumn, because the data of dynamic subcolumn can change after cast.
if (elem->type->hasDynamicSubcolumns() && !elem->type->equals(*requested_column_type))
{
auto casted_column = castColumn({elem->column, elem->type, ""}, requested_column_type);
auto elem_column = requested_column_type->tryGetSubcolumn(subcolumn_name, casted_column);
auto elem_type = requested_column_type->tryGetSubcolumnType(subcolumn_name);
if (!elem_type || !elem_column)
return nullptr;
return elem_column;
}
auto elem_column = elem->type->tryGetSubcolumn(subcolumn_name, elem->column);
auto elem_type = elem->type->tryGetSubcolumnType(subcolumn_name);
if (!elem_type || !elem_column)
return nullptr;
return castColumn({elem_column, elem_type, ""}, requested_subcolumn.type);
}
ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column)
{
auto result_column = tryGetColumnFromBlock(block, requested_column);

1
src/Interpreters/getColumnFromBlock.h
View File

@ -9,5 +9,6 @@ namespace DB
ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column);
ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & requested_column);
ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requested_column_type, const NameAndTypePair & requested_subcolumn);
}

1
src/Loggers/OwnSplitChannel.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <atomic>
#include <memory>
#include <map>
#include <Poco/AutoPtr.h>
#include <Poco/Channel.h>

18
src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp
View File

@ -30,12 +30,15 @@ public:
std::shared_ptr<std::atomic<size_t>> parallel_execution_index_,
InitializerFunc initializer_func_ = {})
: ISource(storage_snapshot->getSampleBlockForColumns(column_names_))
, column_names_and_types(storage_snapshot->getColumnsByNames(
, requested_column_names_and_types(storage_snapshot->getColumnsByNames(
GetColumnsOptions(GetColumnsOptions::All).withSubcolumns().withExtendedObjects(), column_names_))
, data(data_)
, parallel_execution_index(parallel_execution_index_)
, initializer_func(std::move(initializer_func_))
{
auto all_column_names_and_types = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::All).withSubcolumns().withExtendedObjects());
for (const auto & [name, type] : all_column_names_and_types)
all_names_to_types[name] = type;
}
String getName() const override { return "Memory"; }
@ -59,17 +62,20 @@ protected:
const Block & src = (*data)[current_index];
Columns columns;
size_t num_columns = column_names_and_types.size();
size_t num_columns = requested_column_names_and_types.size();
columns.reserve(num_columns);
auto name_and_type = column_names_and_types.begin();
auto name_and_type = requested_column_names_and_types.begin();
for (size_t i = 0; i < num_columns; ++i)
{
if (name_and_type->isSubcolumn())
columns.emplace_back(tryGetSubcolumnFromBlock(src, all_names_to_types[name_and_type->getNameInStorage()], *name_and_type));
else
columns.emplace_back(tryGetColumnFromBlock(src, *name_and_type));
++name_and_type;
}
fillMissingColumns(columns, src.rows(), column_names_and_types, column_names_and_types, {}, nullptr);
fillMissingColumns(columns, src.rows(), requested_column_names_and_types, requested_column_names_and_types, {}, nullptr);
assert(std::all_of(columns.begin(), columns.end(), [](const auto & column) { return column != nullptr; }));
return Chunk(std::move(columns), src.rows());
@ -88,7 +94,9 @@ private:
}
}
const NamesAndTypesList column_names_and_types;
const NamesAndTypesList requested_column_names_and_types;
/// Map (name -> type) for all columns from the storage header.
std::unordered_map<String, DataTypePtr> all_names_to_types;
size_t execution_index = 0;
std::shared_ptr<const Blocks> data;
std::shared_ptr<std::atomic<size_t>> parallel_execution_index;

212
src/Processors/Transforms/WindowTransform.cpp
View File

@ -17,6 +17,9 @@
#include <Common/FieldVisitorConvertToNumber.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <Poco/Logger.h>
#include <Common/logger_useful.h>
#include <limits>
@ -71,6 +74,9 @@ public:
size_t function_index) const = 0;
virtual std::optional<WindowFrame> getDefaultFrame() const { return {}; }
/// Is the frame type supported by this function.
virtual bool checkWindowFrameType(const WindowTransform * /*transform*/) const { return true; }
};
// Compares ORDER BY column values at given rows to find the boundaries of frame:
@ -402,6 +408,19 @@ WindowTransform::WindowTransform(const Block & input_header_,
}
}
}
for (const auto & workspace : workspaces)
{
if (workspace.window_function_impl)
{
if (!workspace.window_function_impl->checkWindowFrameType(this))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported window frame type for function '{}'",
workspace.aggregate_function->getName());
}
}
}
}
WindowTransform::~WindowTransform()
@ -1609,6 +1628,34 @@ struct WindowFunctionHelpers
{
recurrent_detail::setValueToOutputColumn<T>(transform, function_index, value);
}
ALWAYS_INLINE static bool checkPartitionEnterFirstRow(const WindowTransform * transform) { return transform->current_row_number == 1; }
ALWAYS_INLINE static bool checkPartitionEnterLastRow(const WindowTransform * transform)
{
/// This is for fast check.
if (!transform->partition_ended)
return false;
auto current_row = transform->current_row;
/// checkPartitionEnterLastRow is called on each row, also move on current_row.row here.
current_row.row++;
const auto & partition_end_row = transform->partition_end;
/// The partition end is reached, when following is true
/// - current row is the partition end row,
/// - or current row is the last row of all input.
if (current_row != partition_end_row)
{
/// when current row is not the partition end row, we need to check whether it's the last
/// input row.
if (current_row.row < transform->blockRowsNumber(current_row))
return false;
if (partition_end_row.block != current_row.block + 1 || partition_end_row.row)
return false;
}
return true;
}
};
template<typename State>
@ -2058,8 +2105,6 @@ namespace
const WindowTransform * transform,
size_t function_index,
const DataTypes & argument_types);
static void checkWindowFrameType(const WindowTransform * transform);
};
}
@ -2080,6 +2125,29 @@ struct WindowFunctionNtile final : public StatefulWindowFunction<NtileState>
bool allocatesMemoryInArena() const override { return false; }
bool checkWindowFrameType(const WindowTransform * transform) const override
{
if (transform->order_by_indices.empty())
{
LOG_ERROR(getLogger("WindowFunctionNtile"), "Window frame for 'ntile' function must have ORDER BY clause");
return false;
}
// We must wait all for the partition end and get the total rows number in this
// partition. So before the end of this partition, there is no any block could be
// dropped out.
bool is_frame_supported = transform->window_description.frame.begin_type == WindowFrame::BoundaryType::Unbounded
&& transform->window_description.frame.end_type == WindowFrame::BoundaryType::Unbounded;
if (!is_frame_supported)
{
LOG_ERROR(
getLogger("WindowFunctionNtile"),
"Window frame for function 'ntile' should be 'ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'");
return false;
}
return true;
}
std::optional<WindowFrame> getDefaultFrame() const override
{
WindowFrame frame;
@ -2106,7 +2174,6 @@ namespace
{
if (!buckets) [[unlikely]]
{
checkWindowFrameType(transform);
const auto & current_block = transform->blockAt(transform->current_row);
const auto & workspace = transform->workspaces[function_index];
const auto & arg_col = *current_block.original_input_columns[workspace.argument_column_indices[0]];
@ -2128,7 +2195,7 @@ namespace
}
}
// new partition
if (transform->current_row_number == 1) [[unlikely]]
if (WindowFunctionHelpers::checkPartitionEnterFirstRow(transform)) [[unlikely]]
{
current_partition_rows = 0;
current_partition_inserted_row = 0;
@ -2137,25 +2204,9 @@ namespace
current_partition_rows++;
// Only do the action when we meet the last row in this partition.
if (!transform->partition_ended)
if (!WindowFunctionHelpers::checkPartitionEnterLastRow(transform))
return;
else
{
auto current_row = transform->current_row;
current_row.row++;
const auto & end_row = transform->partition_end;
if (current_row != end_row)
{
if (current_row.row < transform->blockRowsNumber(current_row))
return;
if (end_row.block != current_row.block + 1 || end_row.row)
{
return;
}
// else, current_row is the last input row.
}
}
auto bucket_capacity = current_partition_rows / buckets;
auto capacity_diff = current_partition_rows - bucket_capacity * buckets;
@ -2193,24 +2244,116 @@ namespace
bucket_num += 1;
}
}
}
void NtileState::checkWindowFrameType(const WindowTransform * transform)
namespace
{
if (transform->order_by_indices.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Window frame for 'ntile' function must have ORDER BY clause");
struct PercentRankState
{
RowNumber start_row;
UInt64 current_partition_rows = 0;
};
}
// We must wait all for the partition end and get the total rows number in this
// partition. So before the end of this partition, there is no any block could be
// dropped out.
bool is_frame_supported = transform->window_description.frame.begin_type == WindowFrame::BoundaryType::Unbounded
&& transform->window_description.frame.end_type == WindowFrame::BoundaryType::Unbounded;
if (!is_frame_supported)
struct WindowFunctionPercentRank final : public StatefulWindowFunction<PercentRankState>
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Window frame for function 'ntile' should be 'ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING'");
public:
WindowFunctionPercentRank(const std::string & name_,
const DataTypes & argument_types_, const Array & parameters_)
: StatefulWindowFunction(name_, argument_types_, parameters_, std::make_shared<DataTypeFloat64>())
{}
bool allocatesMemoryInArena() const override { return false; }
bool checkWindowFrameType(const WindowTransform * transform) const override
{
if (transform->window_description.frame.type != WindowFrame::FrameType::RANGE
|| transform->window_description.frame.begin_type != WindowFrame::BoundaryType::Unbounded
|| transform->window_description.frame.end_type != WindowFrame::BoundaryType::Current)
{
LOG_ERROR(
getLogger("WindowFunctionPercentRank"),
"Window frame for function 'percent_rank' should be 'RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT'");
return false;
}
return true;
}
std::optional<WindowFrame> getDefaultFrame() const override
{
WindowFrame frame;
frame.type = WindowFrame::FrameType::RANGE;
frame.begin_type = WindowFrame::BoundaryType::Unbounded;
frame.end_type = WindowFrame::BoundaryType::Current;
return frame;
}
void windowInsertResultInto(const WindowTransform * transform, size_t function_index) const override
{
auto & state = getWorkspaceState(transform, function_index);
if (WindowFunctionHelpers::checkPartitionEnterFirstRow(transform))
{
state.current_partition_rows = 0;
state.start_row = transform->current_row;
}
insertRankIntoColumn(transform, function_index);
state.current_partition_rows++;
if (!WindowFunctionHelpers::checkPartitionEnterLastRow(transform))
{
return;
}
UInt64 remaining_rows = state.current_partition_rows;
Float64 percent_rank_denominator = remaining_rows == 1 ? 1 : remaining_rows - 1;
while (remaining_rows > 0)
{
auto block_rows_number = transform->blockRowsNumber(state.start_row);
auto available_block_rows = block_rows_number - state.start_row.row;
if (available_block_rows <= remaining_rows)
{
/// This partition involves multiple blocks. Finish current block and move on to the
/// next block.
auto & to_column = *transform->blockAt(state.start_row).output_columns[function_index];
auto & data = assert_cast<ColumnFloat64 &>(to_column).getData();
for (size_t i = state.start_row.row; i < block_rows_number; ++i)
data[i] = (data[i] - 1) / percent_rank_denominator;
state.start_row.block++;
state.start_row.row = 0;
remaining_rows -= available_block_rows;
}
else
{
/// The partition ends in current block.s
auto & to_column = *transform->blockAt(state.start_row).output_columns[function_index];
auto & data = assert_cast<ColumnFloat64 &>(to_column).getData();
for (size_t i = state.start_row.row, n = state.start_row.row + remaining_rows; i < n; ++i)
{
data[i] = (data[i] - 1) / percent_rank_denominator;
}
state.start_row.row += remaining_rows;
remaining_rows = 0;
}
}
}
inline PercentRankState & getWorkspaceState(const WindowTransform * transform, size_t function_index) const
{
const auto & workspace = transform->workspaces[function_index];
return getState(workspace);
}
inline void insertRankIntoColumn(const WindowTransform * transform, size_t function_index) const
{
auto & to_column = *transform->blockAt(transform->current_row).output_columns[function_index];
assert_cast<ColumnFloat64 &>(to_column).getData().push_back(static_cast<Float64>(transform->peer_group_start_row_number));
}
};
// ClickHouse-specific variant of lag/lead that respects the window frame.
template <bool is_lead>
struct WindowFunctionLagLeadInFrame final : public WindowFunction
@ -2582,6 +2725,13 @@ void registerWindowFunctions(AggregateFunctionFactory & factory)
parameters);
}, properties}, AggregateFunctionFactory::CaseInsensitive);
factory.registerFunction("percent_rank", {[](const std::string & name,
const DataTypes & argument_types, const Array & parameters, const Settings *)
{
return std::make_shared<WindowFunctionPercentRank>(name, argument_types,
parameters);
}, properties}, AggregateFunctionFactory::CaseInsensitive);
factory.registerFunction("row_number", {[](const std::string & name,
const DataTypes & argument_types, const Array & parameters, const Settings *)
{

12
src/Server/TCPHandler.cpp
View File

@ -1875,7 +1875,7 @@ void TCPHandler::receiveQuery()
#endif
}
query_context = session->makeQueryContext(std::move(client_info));
query_context = session->makeQueryContext(client_info);
/// Sets the default database if it wasn't set earlier for the session context.
if (is_interserver_mode && !default_database.empty())
@ -1890,6 +1890,16 @@ void TCPHandler::receiveQuery()
///
/// Settings
///
/// FIXME: Remove when allow_experimental_analyzer will become obsolete.
/// Analyzer became Beta in 24.3 and started to be enabled by default.
/// We have to disable it for ourselves to make sure we don't have different settings on
/// different servers.
if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY
&& client_info.getVersionNumber() < VersionNumber(23, 3, 0)
&& !passed_settings.allow_experimental_analyzer.changed)
passed_settings.set("allow_experimental_analyzer", false);
auto settings_changes = passed_settings.changes();
query_kind = query_context->getClientInfo().query_kind;
if (query_kind == ClientInfo::QueryKind::INITIAL_QUERY)

11
src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -1313,6 +1313,17 @@ void IMergeTreeDataPart::loadRowsCount()
auto buf = metadata_manager->read("count.txt");
readIntText(rows_count, *buf);
assertEOF(*buf);
if (!index_granularity.empty() && rows_count < index_granularity.getTotalRows() && index_granularity_info.fixed_index_granularity)
{
/// Adjust last granule size to match the number of rows in the part in case of fixed index_granularity.
index_granularity.popMark();
index_granularity.appendMark(rows_count % index_granularity_info.fixed_index_granularity);
if (rows_count != index_granularity.getTotalRows())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Index granularity total rows in part {} does not match rows_count: {}, instead of {}",
name, index_granularity.getTotalRows(), rows_count);
}
};
if (index_granularity.empty())

1
src/Storages/MergeTree/IPartMetadataManager.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <memory>
#include <unordered_map>
#include <city.h>
#include <base/types.h>

10
src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp
View File

@ -558,6 +558,9 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai
if (index_granularity_rows != index_granularity.getMarkRows(mark_num))
{
/// With fixed granularity we can have last mark with less rows than granularity
const bool is_last_mark = (mark_num + 1 == index_granularity.getMarksCount());
if (!index_granularity_info.fixed_index_granularity || !is_last_mark)
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Incorrect mark rows for part {} for mark #{}"
@ -821,7 +824,14 @@ void MergeTreeDataPartWriterWide::adjustLastMarkIfNeedAndFlushToDisk(size_t new_
/// Without offset
rows_written_in_last_mark = 0;
}
if (compute_granularity)
{
index_granularity.popMark();
index_granularity.appendMark(new_rows_in_last_mark);
}
}
}
}

253
src/Storages/ObjectStorage/Azure/Configuration.cpp
View File

@ -1,4 +1,5 @@
#include <Storages/ObjectStorage/Azure/Configuration.h>
#include <Poco/URI.h>
#if USE_AZURE_BLOB_STORAGE
@ -40,72 +41,19 @@ const std::unordered_set<std::string_view> optional_configuration_keys = {
"storage_account_url",
};
using AzureClient = Azure::Storage::Blobs::BlobContainerClient;
using AzureClientPtr = std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient>;
namespace
{
bool isConnectionString(const std::string & candidate)
{
return !candidate.starts_with("http");
}
template <typename T>
bool containerExists(T & blob_service_client, const std::string & container_name)
{
Azure::Storage::Blobs::ListBlobContainersOptions options;
options.Prefix = container_name;
options.PageSizeHint = 1;
auto containers_list_response = blob_service_client.ListBlobContainers(options);
auto containers_list = containers_list_response.BlobContainers;
auto it = std::find_if(
containers_list.begin(), containers_list.end(),
[&](const auto & c) { return c.Name == container_name; });
return it != containers_list.end();
}
}
Poco::URI StorageAzureConfiguration::getConnectionURL() const
{
if (!is_connection_string)
return Poco::URI(connection_url);
auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(connection_url);
return Poco::URI(parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl());
}
void StorageAzureConfiguration::check(ContextPtr context) const
{
context->getGlobalContext()->getRemoteHostFilter().checkURL(getConnectionURL());
auto url = Poco::URI(connection_params.getConnectionURL());
context->getGlobalContext()->getRemoteHostFilter().checkURL(url);
Configuration::check(context);
}
StorageAzureConfiguration::StorageAzureConfiguration(const StorageAzureConfiguration & other)
: Configuration(other)
{
connection_url = other.connection_url;
is_connection_string = other.is_connection_string;
account_name = other.account_name;
account_key = other.account_key;
container = other.container;
blob_path = other.blob_path;
blobs_paths = other.blobs_paths;
}
AzureObjectStorage::SettingsPtr StorageAzureConfiguration::createSettings(ContextPtr context)
{
const auto & context_settings = context->getSettingsRef();
auto settings_ptr = std::make_unique<AzureObjectStorageSettings>();
settings_ptr->max_single_part_upload_size = context_settings.azure_max_single_part_upload_size;
settings_ptr->max_single_read_retries = context_settings.azure_max_single_read_retries;
settings_ptr->list_object_keys_size = static_cast<int32_t>(context_settings.azure_list_object_keys_size);
settings_ptr->strict_upload_part_size = context_settings.azure_strict_upload_part_size;
settings_ptr->max_upload_part_size = context_settings.azure_max_upload_part_size;
settings_ptr->max_blocks_in_multipart_upload = context_settings.azure_max_blocks_in_multipart_upload;
settings_ptr->min_upload_part_size = context_settings.azure_min_upload_part_size;
return settings_ptr;
connection_params = other.connection_params;
}
StorageObjectStorage::QuerySettings StorageAzureConfiguration::getQuerySettings(const ContextPtr & context) const
@ -126,174 +74,59 @@ StorageObjectStorage::QuerySettings StorageAzureConfiguration::getQuerySettings(
ObjectStoragePtr StorageAzureConfiguration::createObjectStorage(ContextPtr context, bool is_readonly) /// NOLINT
{
assertInitialized();
auto client = createClient(is_readonly, /* attempt_to_create_container */true);
auto settings = createSettings(context);
auto settings = AzureBlobStorage::getRequestSettings(context->getSettingsRef());
auto client = AzureBlobStorage::getContainerClient(connection_params, is_readonly);
return std::make_unique<AzureObjectStorage>(
"AzureBlobStorage", std::move(client), std::move(settings), container, getConnectionURL().toString());
"AzureBlobStorage",
connection_params.createForContainer(),
std::move(settings),
connection_params.getContainer(),
connection_params.getConnectionURL());
}
AzureClientPtr StorageAzureConfiguration::createClient(bool is_read_only, bool attempt_to_create_container)
static AzureBlobStorage::ConnectionParams getConnectionParams(
const String & connection_url,
const String & container_name,
const std::optional<String> & account_name,
const std::optional<String> & account_key,
const ContextPtr & local_context)
{
using namespace Azure::Storage::Blobs;
AzureBlobStorage::ConnectionParams connection_params;
auto request_settings = AzureBlobStorage::getRequestSettings(local_context->getSettingsRef());
AzureClientPtr result;
if (is_connection_string)
if (account_name && account_key)
{
auto managed_identity_credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
auto blob_service_client = std::make_unique<BlobServiceClient>(BlobServiceClient::CreateFromConnectionString(connection_url));
result = std::make_unique<BlobContainerClient>(BlobContainerClient::CreateFromConnectionString(connection_url, container));
if (attempt_to_create_container)
{
bool container_exists = containerExists(*blob_service_client, container);
if (!container_exists)
{
if (is_read_only)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"AzureBlobStorage container does not exist '{}'",
container);
try
{
result->CreateIfNotExists();
}
catch (const Azure::Storage::StorageException & e)
{
if (!(e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict
&& e.ReasonPhrase == "The specified container already exists."))
{
throw;
}
}
}
}
connection_params.endpoint.storage_account_url = connection_url;
connection_params.endpoint.container_name = container_name;
connection_params.auth_method = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(*account_name, *account_key);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ false);
}
else
{
std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> storage_shared_key_credential;
if (account_name.has_value() && account_key.has_value())
{
storage_shared_key_credential
= std::make_shared<Azure::Storage::StorageSharedKeyCredential>(*account_name, *account_key);
AzureBlobStorage::processURL(connection_url, container_name, connection_params.endpoint, connection_params.auth_method);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ false);
}
std::unique_ptr<BlobServiceClient> blob_service_client;
size_t pos = connection_url.find('?');
std::shared_ptr<Azure::Identity::ManagedIdentityCredential> managed_identity_credential;
if (storage_shared_key_credential)
{
blob_service_client = std::make_unique<BlobServiceClient>(connection_url, storage_shared_key_credential);
}
else
{
/// If conneciton_url does not have '?', then its not SAS
if (pos == std::string::npos)
{
auto workload_identity_credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
blob_service_client = std::make_unique<BlobServiceClient>(connection_url, workload_identity_credential);
}
else
{
managed_identity_credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
blob_service_client = std::make_unique<BlobServiceClient>(connection_url, managed_identity_credential);
}
return connection_params;
}
std::string final_url;
if (pos != std::string::npos)
{
auto url_without_sas = connection_url.substr(0, pos);
final_url = url_without_sas + (url_without_sas.back() == '/' ? "" : "/") + container
+ connection_url.substr(pos);
}
else
final_url
= connection_url + (connection_url.back() == '/' ? "" : "/") + container;
if (!attempt_to_create_container)
{
if (storage_shared_key_credential)
return std::make_unique<BlobContainerClient>(final_url, storage_shared_key_credential);
else
return std::make_unique<BlobContainerClient>(final_url, managed_identity_credential);
}
bool container_exists = containerExists(*blob_service_client, container);
if (container_exists)
{
if (storage_shared_key_credential)
result = std::make_unique<BlobContainerClient>(final_url, storage_shared_key_credential);
else
{
/// If conneciton_url does not have '?', then its not SAS
if (pos == std::string::npos)
{
auto workload_identity_credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
result = std::make_unique<BlobContainerClient>(final_url, workload_identity_credential);
}
else
result = std::make_unique<BlobContainerClient>(final_url, managed_identity_credential);
}
}
else
{
if (is_read_only)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"AzureBlobStorage container does not exist '{}'",
container);
try
{
result = std::make_unique<BlobContainerClient>(blob_service_client->CreateBlobContainer(container).Value);
} catch (const Azure::Storage::StorageException & e)
{
if (e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict
&& e.ReasonPhrase == "The specified container already exists.")
{
if (storage_shared_key_credential)
result = std::make_unique<BlobContainerClient>(final_url, storage_shared_key_credential);
else
{
/// If conneciton_url does not have '?', then its not SAS
if (pos == std::string::npos)
{
auto workload_identity_credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
result = std::make_unique<BlobContainerClient>(final_url, workload_identity_credential);
}
else
result = std::make_unique<BlobContainerClient>(final_url, managed_identity_credential);
}
}
else
{
throw;
}
}
}
}
return result;
}
void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & collection, ContextPtr)
void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & collection, ContextPtr context)
{
validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys);
String connection_url;
String container_name;
std::optional<String> account_name;
std::optional<String> account_key;
if (collection.has("connection_string"))
{
connection_url = collection.get<String>("connection_string");
is_connection_string = true;
}
if (collection.has("storage_account_url"))
{
else if (collection.has("storage_account_url"))
connection_url = collection.get<String>("storage_account_url");
is_connection_string = false;
}
container = collection.get<String>("container");
container_name = collection.get<String>("container");
blob_path = collection.get<String>("blob_path");
if (collection.has("account_name"))
@ -307,6 +140,7 @@ AzureClientPtr StorageAzureConfiguration::createClient(bool is_read_only, bool a
compression_method = collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto"));
blobs_paths = {blob_path};
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
}
void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure)
@ -324,12 +158,14 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
std::unordered_map<std::string_view, size_t> engine_args_to_idx;
connection_url = checkAndGetLiteralArgument<String>(engine_args[0], "connection_string/storage_account_url");
is_connection_string = isConnectionString(connection_url);
container = checkAndGetLiteralArgument<String>(engine_args[1], "container");
String connection_url = checkAndGetLiteralArgument<String>(engine_args[0], "connection_string/storage_account_url");
String container_name = checkAndGetLiteralArgument<String>(engine_args[1], "container");
blob_path = checkAndGetLiteralArgument<String>(engine_args[2], "blobpath");
std::optional<String> account_name;
std::optional<String> account_key;
auto is_format_arg = [] (const std::string & s) -> bool
{
return s == "auto" || FormatFactory::instance().getAllFormats().contains(Poco::toLower(s));
@ -386,7 +222,9 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
account_key = checkAndGetLiteralArgument<String>(engine_args[4], "account_key");
auto sixth_arg = checkAndGetLiteralArgument<String>(engine_args[5], "format/account_name");
if (is_format_arg(sixth_arg))
{
format = sixth_arg;
}
else
{
if (with_structure)
@ -428,6 +266,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
}
blobs_paths = {blob_path};
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
}
void StorageAzureConfiguration::addStructureAndFormatToArgs(

19
src/Storages/ObjectStorage/Azure/Configuration.h
View File

@ -35,8 +35,8 @@ public:
const Paths & getPaths() const override { return blobs_paths; }
void setPaths(const Paths & paths) override { blobs_paths = paths; }
String getNamespace() const override { return container; }
String getDataSourceDescription() const override { return std::filesystem::path(connection_url) / container; }
String getNamespace() const override { return connection_params.getContainer(); }
String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURL()) / connection_params.getContainer(); }
StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const override;
void check(ContextPtr context) const override;
@ -54,22 +54,9 @@ protected:
void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override;
void fromAST(ASTs & args, ContextPtr context, bool with_structure) override;
using AzureClient = Azure::Storage::Blobs::BlobContainerClient;
using AzureClientPtr = std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient>;
std::string connection_url;
bool is_connection_string;
std::optional<std::string> account_name;
std::optional<std::string> account_key;
std::string container;
std::string blob_path;
std::vector<String> blobs_paths;
AzureClientPtr createClient(bool is_read_only, bool attempt_to_create_container);
AzureObjectStorage::SettingsPtr createSettings(ContextPtr local_context);
Poco::URI getConnectionURL() const;
AzureBlobStorage::ConnectionParams connection_params;
};
}

94
src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
View File

@ -65,7 +65,6 @@ StorageObjectStorageSource::StorageObjectStorageSource(
CurrentMetrics::StorageObjectStorageThreadsActive,
CurrentMetrics::StorageObjectStorageThreadsScheduled,
1/* max_threads */))
, columns_desc(info.columns_description)
, file_iterator(file_iterator_)
, schema_cache(StorageObjectStorage::getSchemaCache(context_, configuration->getTypeName()))
, create_reader_scheduler(threadPoolCallbackRunnerUnsafe<ReaderHolder>(*create_reader_pool, "Reader"))
@ -156,20 +155,20 @@ std::shared_ptr<StorageObjectStorageSource::IIterator> StorageObjectStorageSourc
return iterator;
}
void StorageObjectStorageSource::lazyInitialize(size_t processor)
void StorageObjectStorageSource::lazyInitialize()
{
if (initialized)
return;
reader = createReader(processor);
reader = createReader();
if (reader)
reader_future = createReaderAsync(processor);
reader_future = createReaderAsync();
initialized = true;
}
Chunk StorageObjectStorageSource::generate()
{
lazyInitialize(0);
lazyInitialize();
while (true)
{
@ -259,27 +258,30 @@ void StorageObjectStorageSource::addNumRowsToCache(const ObjectInfo & object_inf
schema_cache.addNumRows(cache_key, num_rows);
}
std::optional<size_t> StorageObjectStorageSource::tryGetNumRowsFromCache(const ObjectInfo & object_info)
StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader()
{
const auto cache_key = getKeyForSchemaCache(
getUniqueStoragePathIdentifier(*configuration, object_info),
configuration->format,
format_settings,
getContext());
auto get_last_mod_time = [&]() -> std::optional<time_t>
{
return object_info.metadata
? std::optional<size_t>(object_info.metadata->last_modified.epochTime())
: std::nullopt;
};
return schema_cache.tryGetNumRows(cache_key, get_last_mod_time);
return createReader(
0, file_iterator, configuration, object_storage, read_from_format_info, format_settings,
key_condition, getContext(), &schema_cache, log, max_block_size, max_parsing_threads, need_only_count);
}
StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader(size_t processor)
StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader(
size_t processor,
const std::shared_ptr<IIterator> & file_iterator,
const ConfigurationPtr & configuration,
const ObjectStoragePtr & object_storage,
const ReadFromFormatInfo & read_from_format_info,
const std::optional<FormatSettings> & format_settings,
const std::shared_ptr<const KeyCondition> & key_condition_,
const ContextPtr & context_,
SchemaCache * schema_cache,
const LoggerPtr & log,
size_t max_block_size,
size_t max_parsing_threads,
bool need_only_count)
{
ObjectInfoPtr object_info;
auto query_settings = configuration->getQuerySettings(getContext());
auto query_settings = configuration->getQuerySettings(context_);
do
{
@ -300,9 +302,29 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
std::shared_ptr<ISource> source;
std::unique_ptr<ReadBuffer> read_buf;
auto try_get_num_rows_from_cache = [&]() -> std::optional<size_t>
{
if (!schema_cache)
return std::nullopt;
const auto cache_key = getKeyForSchemaCache(
getUniqueStoragePathIdentifier(*configuration, *object_info),
configuration->format,
format_settings,
context_);
auto get_last_mod_time = [&]() -> std::optional<time_t>
{
return object_info->metadata
? std::optional<size_t>(object_info->metadata->last_modified.epochTime())
: std::nullopt;
};
return schema_cache->tryGetNumRows(cache_key, get_last_mod_time);
};
std::optional<size_t> num_rows_from_cache = need_only_count
&& getContext()->getSettingsRef().use_cache_for_count_from_files
? tryGetNumRowsFromCache(*object_info)
&& context_->getSettingsRef().use_cache_for_count_from_files
? try_get_num_rows_from_cache()
: std::nullopt;
if (num_rows_from_cache)
@ -327,14 +349,14 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
else
{
compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->compression_method);
read_buf = createReadBuffer(*object_info);
read_buf = createReadBuffer(*object_info, object_storage, context_, log);
}
auto input_format = FormatFactory::instance().getInput(
configuration->format,
*read_buf,
read_from_format_info.format_header,
getContext(),
context_,
max_block_size,
format_settings,
need_only_count ? 1 : max_parsing_threads,
@ -343,20 +365,20 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
compression_method,
need_only_count);
if (key_condition)
input_format->setKeyCondition(key_condition);
if (key_condition_)
input_format->setKeyCondition(key_condition_);
if (need_only_count)
input_format->needOnlyCount();
builder.init(Pipe(input_format));
if (columns_desc.hasDefaults())
if (read_from_format_info.columns_description.hasDefaults())
{
builder.addSimpleTransform(
[&](const Block & header)
{
return std::make_shared<AddingDefaultsTransform>(header, columns_desc, *input_format, getContext());
return std::make_shared<AddingDefaultsTransform>(header, read_from_format_info.columns_description, *input_format, context_);
});
}
@ -379,21 +401,25 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade
object_info, std::move(read_buf), std::move(source), std::move(pipeline), std::move(current_reader));
}
std::future<StorageObjectStorageSource::ReaderHolder> StorageObjectStorageSource::createReaderAsync(size_t processor)
std::future<StorageObjectStorageSource::ReaderHolder> StorageObjectStorageSource::createReaderAsync()
{
return create_reader_scheduler([=, this] { return createReader(processor); }, Priority{});
return create_reader_scheduler([=, this] { return createReader(); }, Priority{});
}
std::unique_ptr<ReadBuffer> StorageObjectStorageSource::createReadBuffer(const ObjectInfo & object_info)
std::unique_ptr<ReadBuffer> StorageObjectStorageSource::createReadBuffer(
const ObjectInfo & object_info,
const ObjectStoragePtr & object_storage,
const ContextPtr & context_,
const LoggerPtr & log)
{
const auto & object_size = object_info.metadata->size_bytes;
auto read_settings = getContext()->getReadSettings().adjustBufferSize(object_size);
auto read_settings = context_->getReadSettings().adjustBufferSize(object_size);
read_settings.enable_filesystem_cache = false;
/// FIXME: Changing this setting to default value breaks something around parquet reading
read_settings.remote_read_min_bytes_for_seek = read_settings.remote_fs_buffer_size;
const bool object_too_small = object_size <= 2 * getContext()->getSettings().max_download_buffer_size;
const bool object_too_small = object_size <= 2 * context_->getSettings().max_download_buffer_size;
const bool use_prefetch = object_too_small && read_settings.remote_fs_method == RemoteFSReadMethod::threadpool;
read_settings.remote_fs_method = use_prefetch ? RemoteFSReadMethod::threadpool : RemoteFSReadMethod::read;
/// User's object may change, don't cache it.

30
src/Storages/ObjectStorage/StorageObjectStorageSource.h
View File

@ -76,7 +76,6 @@ protected:
const ReadFromFormatInfo read_from_format_info;
const std::shared_ptr<ThreadPool> create_reader_pool;
ColumnsDescription columns_desc;
std::shared_ptr<IIterator> file_iterator;
SchemaCache & schema_cache;
bool initialized = false;
@ -117,13 +116,32 @@ protected:
std::future<ReaderHolder> reader_future;
/// Recreate ReadBuffer and Pipeline for each file.
ReaderHolder createReader(size_t processor = 0);
std::future<ReaderHolder> createReaderAsync(size_t processor = 0);
std::unique_ptr<ReadBuffer> createReadBuffer(const ObjectInfo & object_info);
static ReaderHolder createReader(
size_t processor,
const std::shared_ptr<IIterator> & file_iterator,
const ConfigurationPtr & configuration,
const ObjectStoragePtr & object_storage,
const ReadFromFormatInfo & read_from_format_info,
const std::optional<FormatSettings> & format_settings,
const std::shared_ptr<const KeyCondition> & key_condition_,
const ContextPtr & context_,
SchemaCache * schema_cache,
const LoggerPtr & log,
size_t max_block_size,
size_t max_parsing_threads,
bool need_only_count);
ReaderHolder createReader();
std::future<ReaderHolder> createReaderAsync();
static std::unique_ptr<ReadBuffer> createReadBuffer(
const ObjectInfo & object_info,
const ObjectStoragePtr & object_storage,
const ContextPtr & context_,
const LoggerPtr & log);
void addNumRowsToCache(const ObjectInfo & object_info, size_t num_rows);
std::optional<size_t> tryGetNumRowsFromCache(const ObjectInfo & object_info);
void lazyInitialize(size_t processor);
void lazyInitialize();
};
class StorageObjectStorageSource::IIterator

10
src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp
View File

@ -62,6 +62,11 @@ void ObjectStorageQueueIFileMetadata::FileStatus::onFailed(const std::string & e
last_exception = exception;
}
void ObjectStorageQueueIFileMetadata::FileStatus::updateState(State state_)
{
state = state_;
}
std::string ObjectStorageQueueIFileMetadata::FileStatus::getException() const
{
std::lock_guard lock(last_exception_mutex);
@ -224,9 +229,14 @@ bool ObjectStorageQueueIFileMetadata::setProcessing()
auto [success, file_state] = setProcessingImpl();
if (success)
{
file_status->onProcessing();
}
else
{
LOG_TEST(log, "Updating state of {} from {} to {}", path, file_status->state.load(), file_state);
file_status->updateState(file_state);
}
LOG_TEST(log, "File {} has state `{}`: will {}process (processing id version: {})",
path, file_state, success ? "" : "not ",

2
src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h
View File

@ -23,7 +23,7 @@ public:
void onProcessing();
void onProcessed();
void onFailed(const std::string & exception);
void updateState(State state_) { state = state_; }
void updateState(State state_);
std::string getException() const;

136
src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp
View File

@ -359,41 +359,38 @@ ObjectStorageQueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t proc
ObjectStorageQueueSource::ObjectStorageQueueSource(
String name_,
size_t processor_id_,
const Block & header_,
std::unique_ptr<StorageObjectStorageSource> internal_source_,
std::shared_ptr<FileIterator> file_iterator_,
ConfigurationPtr configuration_,
ObjectStoragePtr object_storage_,
const ReadFromFormatInfo & read_from_format_info_,
const std::optional<FormatSettings> & format_settings_,
const ObjectStorageQueueSettings & queue_settings_,
std::shared_ptr<ObjectStorageQueueMetadata> files_metadata_,
const ObjectStorageQueueAction & action_,
RemoveFileFunc remove_file_func_,
const NamesAndTypesList & requested_virtual_columns_,
ContextPtr context_,
size_t max_block_size_,
const std::atomic<bool> & shutdown_called_,
const std::atomic<bool> & table_is_being_dropped_,
std::shared_ptr<ObjectStorageQueueLog> system_queue_log_,
const StorageID & storage_id_,
LoggerPtr log_,
size_t max_processed_files_before_commit_,
size_t max_processed_rows_before_commit_,
size_t max_processed_bytes_before_commit_,
size_t max_processing_time_sec_before_commit_,
bool commit_once_processed_)
: ISource(header_)
: ISource(read_from_format_info_.source_header)
, WithContext(context_)
, name(std::move(name_))
, processor_id(processor_id_)
, action(action_)
, file_iterator(file_iterator_)
, configuration(configuration_)
, object_storage(object_storage_)
, read_from_format_info(read_from_format_info_)
, format_settings(format_settings_)
, queue_settings(queue_settings_)
, files_metadata(files_metadata_)
, internal_source(std::move(internal_source_))
, requested_virtual_columns(requested_virtual_columns_)
, max_block_size(max_block_size_)
, shutdown_called(shutdown_called_)
, table_is_being_dropped(table_is_being_dropped_)
, system_queue_log(system_queue_log_)
, storage_id(storage_id_)
, max_processed_files_before_commit(max_processed_files_before_commit_)
, max_processed_rows_before_commit(max_processed_rows_before_commit_)
, max_processed_bytes_before_commit(max_processed_bytes_before_commit_)
, max_processing_time_sec_before_commit(max_processing_time_sec_before_commit_)
, commit_once_processed(commit_once_processed_)
, remove_file_func(remove_file_func_)
, log(log_)
{
}
@ -403,21 +400,6 @@ String ObjectStorageQueueSource::getName() const
return name;
}
void ObjectStorageQueueSource::lazyInitialize(size_t processor)
{
if (initialized)
return;
LOG_TEST(log, "Initializing a new reader");
internal_source->lazyInitialize(processor);
reader = std::move(internal_source->reader);
if (reader)
reader_future = std::move(internal_source->reader_future);
initialized = true;
}
Chunk ObjectStorageQueueSource::generate()
{
Chunk chunk;
@ -442,19 +424,33 @@ Chunk ObjectStorageQueueSource::generate()
Chunk ObjectStorageQueueSource::generateImpl()
{
lazyInitialize(processor_id);
while (true)
{
if (!reader)
{
if (shutdown_called)
{
LOG_TEST(log, "Shutdown called");
break;
}
const auto context = getContext();
reader = StorageObjectStorageSource::createReader(
processor_id, file_iterator, configuration, object_storage, read_from_format_info,
format_settings, nullptr, context, nullptr, log, max_block_size,
context->getSettingsRef().max_parsing_threads.value, /* need_only_count */false);
if (!reader)
{
LOG_TEST(log, "No reader");
break;
}
}
const auto * object_info = dynamic_cast<const ObjectStorageQueueObjectInfo *>(reader.getObjectInfo().get());
auto file_metadata = object_info->file_metadata;
auto file_status = file_metadata->getFileStatus();
const auto & path = reader.getObjectInfo()->getPath();
if (isCancelled())
{
@ -477,8 +473,6 @@ Chunk ObjectStorageQueueSource::generateImpl()
break;
}
const auto & path = reader.getObjectInfo()->getPath();
if (shutdown_called)
{
LOG_TEST(log, "Shutdown called");
@ -526,7 +520,7 @@ Chunk ObjectStorageQueueSource::generateImpl()
total_processed_bytes += chunk.bytes();
VirtualColumnUtils::addRequestedFileLikeStorageVirtualsToChunk(
chunk, requested_virtual_columns,
chunk, read_from_format_info.requested_virtual_columns,
{
.path = path,
.size = reader.getObjectInfo()->metadata->size_bytes
@ -545,9 +539,6 @@ Chunk ObjectStorageQueueSource::generateImpl()
if (processed_rows_from_file == 0)
{
auto * file_iterator = dynamic_cast<FileIterator *>(internal_source->file_iterator.get());
chassert(file_iterator);
if (file_status->retries < file_metadata->getMaxTries())
file_iterator->returnForRetry(reader.getObjectInfo());
@ -562,11 +553,13 @@ Chunk ObjectStorageQueueSource::generateImpl()
file_status->setProcessingEndTime();
file_status.reset();
reader = {};
processed_rows_from_file = 0;
processed_files.push_back(file_metadata);
if (processed_files.size() == max_processed_files_before_commit)
if (queue_settings.max_processed_files_before_commit
&& processed_files.size() == queue_settings.max_processed_files_before_commit)
{
LOG_TRACE(log, "Number of max processed files before commit reached "
"(rows: {}, bytes: {}, files: {})",
@ -574,67 +567,29 @@ Chunk ObjectStorageQueueSource::generateImpl()
break;
}
bool rows_or_bytes_or_time_limit_reached = false;
if (max_processed_rows_before_commit
&& total_processed_rows == max_processed_rows_before_commit)
if (queue_settings.max_processed_rows_before_commit
&& total_processed_rows == queue_settings.max_processed_rows_before_commit)
{
LOG_TRACE(log, "Number of max processed rows before commit reached "
"(rows: {}, bytes: {}, files: {})",
total_processed_rows, total_processed_bytes, processed_files.size());
rows_or_bytes_or_time_limit_reached = true;
break;
}
else if (max_processed_bytes_before_commit
&& total_processed_bytes == max_processed_bytes_before_commit)
else if (queue_settings.max_processed_bytes_before_commit
&& total_processed_bytes == queue_settings.max_processed_bytes_before_commit)
{
LOG_TRACE(log, "Number of max processed bytes before commit reached "
"(rows: {}, bytes: {}, files: {})",
total_processed_rows, total_processed_bytes, processed_files.size());
rows_or_bytes_or_time_limit_reached = true;
break;
}
else if (max_processing_time_sec_before_commit
&& total_stopwatch.elapsedSeconds() >= max_processing_time_sec_before_commit)
else if (queue_settings.max_processing_time_sec_before_commit
&& total_stopwatch.elapsedSeconds() >= queue_settings.max_processing_time_sec_before_commit)
{
LOG_TRACE(log, "Max processing time before commit reached "
"(rows: {}, bytes: {}, files: {})",
total_processed_rows, total_processed_bytes, processed_files.size());
rows_or_bytes_or_time_limit_reached = true;
}
if (rows_or_bytes_or_time_limit_reached)
{
if (!reader_future.valid())
break;
LOG_TRACE(log, "Rows or bytes limit reached, but we have one more file scheduled already, "
"will process it despite the limit");
}
if (shutdown_called)
{
LOG_TRACE(log, "Shutdown was called, stopping sync");
break;
}
chassert(reader_future.valid());
reader = reader_future.get();
if (!reader)
{
LOG_TEST(log, "Reader finished");
break;
}
file_status = files_metadata->getFileStatus(reader.getObjectInfo()->getPath());
if (!rows_or_bytes_or_time_limit_reached && processed_files.size() + 1 < max_processed_files_before_commit)
{
/// Even if task is finished the thread may be not freed in pool.
/// So wait until it will be freed before scheduling a new task.
internal_source->create_reader_pool->wait();
reader_future = internal_source->createReaderAsync(processor_id);
}
}
@ -679,12 +634,11 @@ void ObjectStorageQueueSource::commit(bool success, const std::string & exceptio
void ObjectStorageQueueSource::applyActionAfterProcessing(const String & path)
{
switch (action)
switch (queue_settings.after_processing.value)
{
case ObjectStorageQueueAction::DELETE:
{
assert(remove_file_func);
remove_file_func(path);
object_storage->removeObject(StoredObject(path));
break;
}
case ObjectStorageQueueAction::KEEP:

37
src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h
View File

@ -21,7 +21,6 @@ class ObjectStorageQueueSource : public ISource, WithContext
public:
using Storage = StorageObjectStorage;
using Source = StorageObjectStorageSource;
using RemoveFileFunc = std::function<void(std::string)>;
using BucketHolderPtr = ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr;
using BucketHolder = ObjectStorageQueueOrderedFileMetadata::BucketHolder;
@ -97,22 +96,20 @@ public:
ObjectStorageQueueSource(
String name_,
size_t processor_id_,
const Block & header_,
std::unique_ptr<StorageObjectStorageSource> internal_source_,
std::shared_ptr<FileIterator> file_iterator_,
ConfigurationPtr configuration_,
ObjectStoragePtr object_storage_,
const ReadFromFormatInfo & read_from_format_info_,
const std::optional<FormatSettings> & format_settings_,
const ObjectStorageQueueSettings & queue_settings_,
std::shared_ptr<ObjectStorageQueueMetadata> files_metadata_,
const ObjectStorageQueueAction & action_,
RemoveFileFunc remove_file_func_,
const NamesAndTypesList & requested_virtual_columns_,
ContextPtr context_,
size_t max_block_size_,
const std::atomic<bool> & shutdown_called_,
const std::atomic<bool> & table_is_being_dropped_,
std::shared_ptr<ObjectStorageQueueLog> system_queue_log_,
const StorageID & storage_id_,
LoggerPtr log_,
size_t max_processed_files_before_commit_,
size_t max_processed_rows_before_commit_,
size_t max_processed_bytes_before_commit_,
size_t max_processing_time_sec_before_commit_,
bool commit_once_processed_);
static Block getHeader(Block sample_block, const std::vector<NameAndTypePair> & requested_virtual_columns);
@ -128,29 +125,27 @@ public:
private:
const String name;
const size_t processor_id;
const ObjectStorageQueueAction action;
const std::shared_ptr<FileIterator> file_iterator;
const ConfigurationPtr configuration;
const ObjectStoragePtr object_storage;
const ReadFromFormatInfo read_from_format_info;
const std::optional<FormatSettings> format_settings;
const ObjectStorageQueueSettings queue_settings;
const std::shared_ptr<ObjectStorageQueueMetadata> files_metadata;
const std::shared_ptr<StorageObjectStorageSource> internal_source;
const NamesAndTypesList requested_virtual_columns;
const size_t max_block_size;
const std::atomic<bool> & shutdown_called;
const std::atomic<bool> & table_is_being_dropped;
const std::shared_ptr<ObjectStorageQueueLog> system_queue_log;
const StorageID storage_id;
const size_t max_processed_files_before_commit;
const size_t max_processed_rows_before_commit;
const size_t max_processed_bytes_before_commit;
const size_t max_processing_time_sec_before_commit;
const bool commit_once_processed;
RemoveFileFunc remove_file_func;
LoggerPtr log;
std::vector<ObjectStorageQueueMetadata::FileMetadataPtr> processed_files;
std::vector<ObjectStorageQueueMetadata::FileMetadataPtr> failed_during_read_files;
Source::ReaderHolder reader;
std::future<Source::ReaderHolder> reader_future;
std::atomic<bool> initialized{false};
size_t processed_rows_from_file = 0;
size_t total_processed_rows = 0;
@ -165,8 +160,6 @@ private:
ObjectStorageQueueMetadata::FileStatus & file_status_,
size_t processed_rows,
bool processed);
void lazyInitialize(size_t processor);
};
}

41
src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
View File

@ -352,43 +352,14 @@ std::shared_ptr<ObjectStorageQueueSource> StorageObjectStorageQueue::createSourc
ContextPtr local_context,
bool commit_once_processed)
{
auto internal_source = std::make_unique<StorageObjectStorageSource>(
getName(),
object_storage,
configuration,
info,
format_settings,
local_context,
max_block_size,
file_iterator,
local_context->getSettingsRef().max_download_threads,
false);
auto file_deleter = [=, this](const std::string & path) mutable
{
object_storage->removeObject(StoredObject(path));
};
return std::make_shared<ObjectStorageQueueSource>(
getName(),
processor_id,
info.source_header,
std::move(internal_source),
files_metadata,
queue_settings->after_processing,
file_deleter,
info.requested_virtual_columns,
local_context,
shutdown_called,
table_is_being_dropped,
getName(), processor_id,
file_iterator, configuration, object_storage,
info, format_settings,
*queue_settings, files_metadata,
local_context, max_block_size, shutdown_called, table_is_being_dropped,
getQueueLog(object_storage, local_context, *queue_settings),
getStorageID(),
log,
queue_settings->max_processed_files_before_commit,
queue_settings->max_processed_rows_before_commit,
queue_settings->max_processed_bytes_before_commit,
queue_settings->max_processing_time_sec_before_commit,
commit_once_processed);
getStorageID(), log, commit_once_processed);
}
bool StorageObjectStorageQueue::hasDependencies(const StorageID & table_id)

169
src/Storages/StorageFuzzQuery.cpp Normal file
View File

@ -0,0 +1,169 @@
#include <Storages/StorageFuzzQuery.h>
#include <optional>
#include <unordered_set>
#include <Columns/ColumnString.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Storages/NamedCollectionsHelpers.h>
#include <Storages/StorageFactory.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <Parsers/ParserQuery.h>
#include <Parsers/parseQuery.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
ColumnPtr FuzzQuerySource::createColumn()
{
auto column = ColumnString::create();
ColumnString::Chars & data_to = column->getChars();
ColumnString::Offsets & offsets_to = column->getOffsets();
offsets_to.resize(block_size);
IColumn::Offset offset = 0;
auto fuzz_base = query;
size_t row_num = 0;
while (row_num < block_size)
{
ASTPtr new_query = fuzz_base->clone();
auto base_before_fuzz = fuzz_base->formatForErrorMessage();
fuzzer.fuzzMain(new_query);
auto fuzzed_text = new_query->formatForErrorMessage();
if (base_before_fuzz == fuzzed_text)
continue;
/// AST is too long, will start from the original query.
if (config.max_query_length > 500)
{
fuzz_base = query;
continue;
}
IColumn::Offset next_offset = offset + fuzzed_text.size() + 1;
data_to.resize(next_offset);
std::copy(fuzzed_text.begin(), fuzzed_text.end(), &data_to[offset]);
data_to[offset + fuzzed_text.size()] = 0;
offsets_to[row_num] = next_offset;
offset = next_offset;
fuzz_base = new_query;
++row_num;
}
return column;
}
StorageFuzzQuery::StorageFuzzQuery(
const StorageID & table_id_, const ColumnsDescription & columns_, const String & comment_, const Configuration & config_)
: IStorage(table_id_), config(config_)
{
StorageInMemoryMetadata storage_metadata;
storage_metadata.setColumns(columns_);
storage_metadata.setComment(comment_);
setInMemoryMetadata(storage_metadata);
}
Pipe StorageFuzzQuery::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & /*query_info*/,
ContextPtr /*context*/,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
size_t num_streams)
{
storage_snapshot->check(column_names);
Pipes pipes;
pipes.reserve(num_streams);
const ColumnsDescription & our_columns = storage_snapshot->metadata->getColumns();
Block block_header;
for (const auto & name : column_names)
{
const auto & name_type = our_columns.get(name);
MutableColumnPtr column = name_type.type->createColumn();
block_header.insert({std::move(column), name_type.type, name_type.name});
}
const char * begin = config.query.data();
const char * end = begin + config.query.size();
ParserQuery parser(end, false);
auto query = parseQuery(parser, begin, end, "", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
for (UInt64 i = 0; i < num_streams; ++i)
pipes.emplace_back(std::make_shared<FuzzQuerySource>(max_block_size, block_header, config, query));
return Pipe::unitePipes(std::move(pipes));
}
StorageFuzzQuery::Configuration StorageFuzzQuery::getConfiguration(ASTs & engine_args, ContextPtr local_context)
{
StorageFuzzQuery::Configuration configuration{};
// Supported signatures:
//
// FuzzQuery(query)
// FuzzQuery(query, max_query_length)
// FuzzQuery(query, max_query_length, random_seed)
if (engine_args.empty() || engine_args.size() > 3)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "FuzzQuery requires 1 to 3 arguments: query, max_query_length, random_seed");
for (auto & engine_arg : engine_args)
engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context);
auto first_arg = checkAndGetLiteralArgument<String>(engine_args[0], "query");
configuration.query = std::move(first_arg);
if (engine_args.size() >= 2)
{
const auto & literal = engine_args[1]->as<const ASTLiteral &>();
if (!literal.value.isNull())
configuration.max_query_length = checkAndGetLiteralArgument<UInt64>(literal, "max_query_length");
}
if (engine_args.size() == 3)
{
const auto & literal = engine_args[2]->as<const ASTLiteral &>();
if (!literal.value.isNull())
configuration.random_seed = checkAndGetLiteralArgument<UInt64>(literal, "random_seed");
}
return configuration;
}
void registerStorageFuzzQuery(StorageFactory & factory)
{
factory.registerStorage(
"FuzzQuery",
[](const StorageFactory::Arguments & args) -> std::shared_ptr<StorageFuzzQuery>
{
ASTs & engine_args = args.engine_args;
if (engine_args.empty())
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage FuzzQuery must have arguments.");
StorageFuzzQuery::Configuration configuration = StorageFuzzQuery::getConfiguration(engine_args, args.getLocalContext());
for (const auto& col : args.columns)
if (col.type->getTypeId() != TypeIndex::String)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "'StorageFuzzQuery' supports only columns of String type, got {}.", col.type->getName());
return std::make_shared<StorageFuzzQuery>(args.table_id, args.columns, args.comment, configuration);
});
}
}

88
src/Storages/StorageFuzzQuery.h Normal file
View File

@ -0,0 +1,88 @@
#pragma once
#include <Storages/IStorage.h>
#include <Storages/StorageConfiguration.h>
#include <Common/randomSeed.h>
#include <Common/QueryFuzzer.h>
#include "config.h"
namespace DB
{
class NamedCollection;
class StorageFuzzQuery final : public IStorage
{
public:
struct Configuration : public StatelessTableEngineConfiguration
{
String query;
UInt64 max_query_length = 500;
UInt64 random_seed = randomSeed();
};
StorageFuzzQuery(
const StorageID & table_id_, const ColumnsDescription & columns_, const String & comment_, const Configuration & config_);
std::string getName() const override { return "FuzzQuery"; }
Pipe read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & query_info,
ContextPtr context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
size_t num_streams) override;
static StorageFuzzQuery::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context);
private:
const Configuration config;
};
class FuzzQuerySource : public ISource
{
public:
FuzzQuerySource(
UInt64 block_size_, Block block_header_, const StorageFuzzQuery::Configuration & config_, ASTPtr query_)
: ISource(block_header_)
, block_size(block_size_)
, block_header(std::move(block_header_))
, config(config_)
, query(query_)
, fuzzer(config_.random_seed)
{
}
String getName() const override { return "FuzzQuery"; }
protected:
Chunk generate() override
{
Columns columns;
columns.reserve(block_header.columns());
for (const auto & col : block_header)
{
chassert(col.type->getTypeId() == TypeIndex::String);
columns.emplace_back(createColumn());
}
return {std::move(columns), block_size};
}
private:
ColumnPtr createColumn();
UInt64 block_size;
Block block_header;
StorageFuzzQuery::Configuration config;
ASTPtr query;
QueryFuzzer fuzzer;
};
}

1
src/Storages/StorageMaterializedView.cpp
View File

@ -161,6 +161,7 @@ StorageMaterializedView::StorageMaterializedView(
manual_create_query->setDatabase(getStorageID().database_name);
manual_create_query->setTable(generateInnerTableName(getStorageID()));
manual_create_query->uuid = query.to_inner_uuid;
manual_create_query->has_uuid = query.to_inner_uuid != UUIDHelpers::Nil;
auto new_columns_list = std::make_shared<ASTColumns>();
new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr());

2
src/Storages/registerStorages.cpp
View File

@ -26,6 +26,7 @@ void registerStorageGenerateRandom(StorageFactory & factory);
void registerStorageExecutable(StorageFactory & factory);
void registerStorageWindowView(StorageFactory & factory);
void registerStorageLoop(StorageFactory & factory);
void registerStorageFuzzQuery(StorageFactory & factory);
#if USE_RAPIDJSON || USE_SIMDJSON
void registerStorageFuzzJSON(StorageFactory & factory);
#endif
@ -123,6 +124,7 @@ void registerStorages()
registerStorageExecutable(factory);
registerStorageWindowView(factory);
registerStorageLoop(factory);
registerStorageFuzzQuery(factory);
#if USE_RAPIDJSON || USE_SIMDJSON
registerStorageFuzzJSON(factory);
#endif

54
src/TableFunctions/TableFunctionFuzzQuery.cpp Normal file
View File

@ -0,0 +1,54 @@
#include <TableFunctions/TableFunctionFuzzQuery.h>
#include <DataTypes/DataTypeString.h>
#include <Storages/checkAndGetLiteralArgument.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <TableFunctions/registerTableFunctions.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
void TableFunctionFuzzQuery::parseArguments(const ASTPtr & ast_function, ContextPtr context)
{
ASTs & args_func = ast_function->children;
if (args_func.size() != 1)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName());
auto args = args_func.at(0)->children;
configuration = StorageFuzzQuery::getConfiguration(args, context);
}
StoragePtr TableFunctionFuzzQuery::executeImpl(
const ASTPtr & /*ast_function*/,
ContextPtr context,
const std::string & table_name,
ColumnsDescription /*cached_columns*/,
bool is_insert_query) const
{
ColumnsDescription columns = getActualTableStructure(context, is_insert_query);
auto res = std::make_shared<StorageFuzzQuery>(
StorageID(getDatabaseName(), table_name),
columns,
/* comment */ String{},
configuration);
res->startup();
return res;
}
void registerTableFunctionFuzzQuery(TableFunctionFactory & factory)
{
factory.registerFunction<TableFunctionFuzzQuery>(
{.documentation
= {.description = "Perturbs a query string with random variations.",
.returned_value = "A table object with a single column containing perturbed query strings."},
.allow_readonly = true});
}
}

42
src/TableFunctions/TableFunctionFuzzQuery.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include <optional>
#include <TableFunctions/ITableFunction.h>
#include <DataTypes/DataTypeString.h>
#include <Storages/StorageFuzzQuery.h>
#include "config.h"
namespace DB
{
class TableFunctionFuzzQuery : public ITableFunction
{
public:
static constexpr auto name = "fuzzQuery";
std::string getName() const override { return name; }
void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
ColumnsDescription getActualTableStructure(ContextPtr /* context */, bool /* is_insert_query */) const override
{
return ColumnsDescription{{"query", std::make_shared<DataTypeString>()}};
}
private:
StoragePtr executeImpl(
const ASTPtr & ast_function,
ContextPtr context,
const std::string & table_name,
ColumnsDescription cached_columns,
bool is_insert_query) const override;
const char * getStorageTypeName() const override { return "fuzzQuery"; }
String source;
std::optional<UInt64> random_seed;
StorageFuzzQuery::Configuration configuration;
};
}

1
src/TableFunctions/registerTableFunctions.cpp
View File

@ -26,6 +26,7 @@ void registerTableFunctions()
registerTableFunctionMongoDB(factory);
registerTableFunctionRedis(factory);
registerTableFunctionMergeTreeIndex(factory);
registerTableFunctionFuzzQuery(factory);
#if USE_RAPIDJSON || USE_SIMDJSON
registerTableFunctionFuzzJSON(factory);
#endif

1
src/TableFunctions/registerTableFunctions.h
View File

@ -23,6 +23,7 @@ void registerTableFunctionGenerate(TableFunctionFactory & factory);
void registerTableFunctionMongoDB(TableFunctionFactory & factory);
void registerTableFunctionRedis(TableFunctionFactory & factory);
void registerTableFunctionMergeTreeIndex(TableFunctionFactory & factory);
void registerTableFunctionFuzzQuery(TableFunctionFactory & factory);
#if USE_RAPIDJSON || USE_SIMDJSON
void registerTableFunctionFuzzJSON(TableFunctionFactory & factory);
#endif

3
src/configure_config.cmake
View File

@ -170,5 +170,8 @@ endif()
if (TARGET ch_contrib::pocketfft)
set(USE_POCKETFFT 1)
endif()
if (TARGET ch_contrib::prometheus_protobufs)
set(USE_PROMETHEUS_PROTOBUFS 1)
endif()
set(SOURCE_DIR ${PROJECT_SOURCE_DIR})

7
tests/ci/integration_tests_runner.py
View File

@ -434,7 +434,14 @@ class ClickhouseIntegrationTestsRunner:
"Getting all tests to the file %s with cmd: \n%s", out_file_full, cmd
)
with open(out_file_full, "wb") as ofd:
try:
subprocess.check_call(cmd, shell=True, stdout=ofd, stderr=ofd)
except subprocess.CalledProcessError as ex:
print("ERROR: Setting test plan failed. Output:")
with open(out_file_full, "r", encoding="utf-8") as file:
for line in file:
print(" " + line, end="")
raise ex
all_tests = set()
with open(out_file_full, "r", encoding="utf-8") as all_tests_fd:

5
tests/ci/sync_pr.py
View File

@ -101,19 +101,16 @@ def main():
assert pr_info.merged_pr, "BUG. merged PR number could not been determined"
prs = gh.get_pulls_from_search(
query=f"head:sync-upstream/pr/{pr_info.merged_pr} org:ClickHouse type:pr",
query=f"head:sync-upstream/pr/{pr_info.merged_pr} org:ClickHouse type:pr is:open",
repo="ClickHouse/clickhouse-private",
)
sync_pr = None
if len(prs) > 1:
print(f"WARNING: More than one PR found [{prs}] - exiting")
elif len(prs) == 0:
print("WARNING: No Sync PR found")
else:
sync_pr = prs[0]
if args.merge:
merge_sync_pr(gh, sync_pr)
elif args.status:

2
tests/integration/helpers/cluster.py
View File

@ -73,7 +73,7 @@ CLICKHOUSE_ERROR_LOG_FILE = "/var/log/clickhouse-server/clickhouse-server.err.lo
# Minimum version we use in integration tests to check compatibility with old releases
# Keep in mind that we only support upgrading between releases that are at most 1 year different.
# This means that this minimum need to be, at least, 1 year older than the current release
CLICKHOUSE_CI_MIN_TESTED_VERSION = "22.8"
CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3"
# to create docker-compose env file

0
tests/integration/test_analyzer_compatibility/__init__.py Normal file
View File

17
tests/integration/test_analyzer_compatibility/configs/remote_servers.xml Normal file
View File

@ -0,0 +1,17 @@
<clickhouse>
<remote_servers>
<test_cluster_mixed>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>current</host>
<port>9000</port>
</replica>
<replica>
<host>backward</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_mixed>
</remote_servers>
</clickhouse>

100
tests/integration/test_analyzer_compatibility/test.py Normal file
View File

@ -0,0 +1,100 @@
import uuid
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
CLICKHOUSE_MAX_VERSION_WITH_ANALYZER_DISABLED_BY_DEFAULT = "24.2"
cluster = ClickHouseCluster(__file__)
# Here analyzer is enabled by default
current = cluster.add_instance(
"current",
main_configs=["configs/remote_servers.xml"],
)
# Here analyzer is disabled by default
backward = cluster.add_instance(
"backward",
use_old_analyzer=True,
main_configs=["configs/remote_servers.xml"],
image="clickhouse/clickhouse-server",
tag=CLICKHOUSE_MAX_VERSION_WITH_ANALYZER_DISABLED_BY_DEFAULT,
with_installed_binary=True,
)
@pytest.fixture(scope="module")
def start_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def test_two_new_versions(start_cluster):
# Two new versions (both know about the analyzer)
# One have it enabled by default, another one - disabled.
current.query("SYSTEM FLUSH LOGS")
backward.query("SYSTEM FLUSH LOGS")
query_id = str(uuid.uuid4())
current.query(
"SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables);",
query_id=query_id,
)
current.query("SYSTEM FLUSH LOGS")
backward.query("SYSTEM FLUSH LOGS")
assert (
current.query(
"""
SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
FROM clusterAllReplicas('test_cluster_mixed', system.one)
ORDER BY h;"""
)
== TSV([["backward", "true"], ["current", "true"]])
)
# Should be enabled everywhere
analyzer_enabled = current.query(
f"""
SELECT
DISTINCT Settings['allow_experimental_analyzer']
FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
WHERE initial_query_id = '{query_id}';"""
)
assert TSV(analyzer_enabled) == TSV("1")
query_id = str(uuid.uuid4())
backward.query(
"SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables)",
query_id=query_id,
)
current.query("SYSTEM FLUSH LOGS")
backward.query("SYSTEM FLUSH LOGS")
assert (
backward.query(
"""
SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
FROM clusterAllReplicas('test_cluster_mixed', system.one)
ORDER BY h;"""
)
== TSV([["backward", "false"], ["current", "false"]])
)
# Should be disabled everywhere
analyzer_enabled = backward.query(
f"""
SELECT
DISTINCT Settings['allow_experimental_analyzer']
FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
WHERE initial_query_id = '{query_id}';"""
)
assert TSV(analyzer_enabled) == TSV("0")

18
tests/integration/test_backward_compatibility/test_functions.py
View File

@ -130,10 +130,13 @@ def test_string_functions(start_cluster):
functions = map(lambda x: x.strip(), functions)
excludes = [
# The argument of this function is not a seed, but an arbitrary expression needed for bypassing common subexpression elimination.
"rand",
"rand64",
"randConstant",
"randCanonical",
"generateUUIDv4",
"generateULID",
# Syntax error otherwise
"position",
"substring",
@ -153,6 +156,18 @@ def test_string_functions(start_cluster):
"tryBase64Decode",
# Removed in 23.9
"meiliMatch",
# These functions require more than one argument.
"parseDateTimeInJodaSyntaxOrZero",
"parseDateTimeInJodaSyntaxOrNull",
"parseDateTimeOrNull",
"parseDateTimeOrZero",
"parseDateTime",
# The argument is effectively a disk name (and we don't have one with name foo)
"filesystemUnreserved",
"filesystemCapacity",
"filesystemAvailable",
# Exclude it for now. Looks like the result depends on the build type.
"farmHash64",
]
functions = filter(lambda x: x not in excludes, functions)
@ -205,6 +220,9 @@ def test_string_functions(start_cluster):
# Function X takes exactly one parameter:
# The function 'X' can only be used as a window function
"BAD_ARGUMENTS",
# String foo is obviously not a valid IP address.
"CANNOT_PARSE_IPV4",
"CANNOT_PARSE_IPV6",
]
if any(map(lambda x: x in error_message, allowed_errors)):
logging.info("Skipping %s", function)

35
tests/integration/test_distributed_inter_server_secret/test.py
View File

@ -7,7 +7,7 @@ import uuid
import time
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
@ -27,9 +27,6 @@ def make_instance(name, *args, **kwargs):
)
# DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 added in 23.3, ensure that CLICKHOUSE_CI_MIN_TESTED_VERSION fits
assert CLICKHOUSE_CI_MIN_TESTED_VERSION < "23.3"
# _n1/_n2 contains cluster with different <secret> -- should fail
# only n1 contains new_user
n1 = make_instance(
@ -38,14 +35,6 @@ n1 = make_instance(
user_configs=["configs/users.d/new_user.xml"],
)
n2 = make_instance("n2", main_configs=["configs/remote_servers_n2.xml"])
backward = make_instance(
"backward",
main_configs=["configs/remote_servers_backward.xml"],
image="clickhouse/clickhouse-server",
# version without DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2
tag=CLICKHOUSE_CI_MIN_TESTED_VERSION,
with_installed_binary=True,
)
users = pytest.mark.parametrize(
"user,password",
@ -427,28 +416,6 @@ def test_per_user_protocol_settings_secure_cluster(user, password):
)
@users
def test_user_secure_cluster_with_backward(user, password):
id_ = "with-backward-query-dist_secure-" + user
n1.query(
f"SELECT *, '{id_}' FROM dist_secure_backward", user=user, password=password
)
assert get_query_user_info(n1, id_) == [user, user]
assert get_query_user_info(backward, id_) == [user, user]
@users
def test_user_secure_cluster_from_backward(user, password):
id_ = "from-backward-query-dist_secure-" + user
backward.query(f"SELECT *, '{id_}' FROM dist_secure", user=user, password=password)
assert get_query_user_info(n1, id_) == [user, user]
assert get_query_user_info(backward, id_) == [user, user]
assert n1.contains_in_log(
"Using deprecated interserver protocol because the client is too old. Consider upgrading all nodes in cluster."
)
def test_secure_cluster_distributed_over_distributed_different_users():
# This works because we will have initial_user='default'
n1.query(

Some files were not shown because too many files have changed in this diff Show More