mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into allow_atomic_database_inside_materialize_mysql
This commit is contained in:
commit
41e99cf261
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/types.h>
|
||||
#include <common/extended_types.h>
|
||||
|
||||
namespace common
|
||||
{
|
||||
|
108
base/common/extended_types.h
Normal file
108
base/common/extended_types.h
Normal file
@ -0,0 +1,108 @@
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include <common/types.h>
|
||||
#include <common/wide_integer.h>
|
||||
|
||||
using Int128 = __int128;
|
||||
|
||||
using wInt256 = wide::integer<256, signed>;
|
||||
using wUInt256 = wide::integer<256, unsigned>;
|
||||
|
||||
static_assert(sizeof(wInt256) == 32);
|
||||
static_assert(sizeof(wUInt256) == 32);
|
||||
|
||||
/// The standard library type traits, such as std::is_arithmetic, with one exception
|
||||
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
|
||||
/// So instead of using the std type_traits, we use our own version which allows extension.
|
||||
template <typename T>
|
||||
struct is_signed
|
||||
{
|
||||
static constexpr bool value = std::is_signed_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_signed<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_signed_v = is_signed<T>::value;
|
||||
|
||||
template <typename T>
|
||||
struct is_unsigned
|
||||
{
|
||||
static constexpr bool value = std::is_unsigned_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
|
||||
|
||||
|
||||
/// TODO: is_integral includes char, char8_t and wchar_t.
|
||||
template <typename T>
|
||||
struct is_integer
|
||||
{
|
||||
static constexpr bool value = std::is_integral_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_integer<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_integer_v = is_integer<T>::value;
|
||||
|
||||
|
||||
template <typename T>
|
||||
struct is_arithmetic
|
||||
{
|
||||
static constexpr bool value = std::is_arithmetic_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
|
||||
template <typename T>
|
||||
struct make_unsigned
|
||||
{
|
||||
typedef std::make_unsigned_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
|
||||
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
|
||||
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
|
||||
|
||||
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
|
||||
|
||||
template <typename T>
|
||||
struct make_signed
|
||||
{
|
||||
typedef std::make_signed_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_signed<wInt256> { using type = wInt256; };
|
||||
template <> struct make_signed<wUInt256> { using type = wInt256; };
|
||||
|
||||
template <typename T> using make_signed_t = typename make_signed<T>::type;
|
||||
|
||||
template <typename T>
|
||||
struct is_big_int
|
||||
{
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_big_int_v = is_big_int<T>::value;
|
||||
|
||||
template <typename To, typename From>
|
||||
inline To bigint_cast(const From & x [[maybe_unused]])
|
||||
{
|
||||
return static_cast<To>(x);
|
||||
}
|
@ -2,9 +2,6 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
|
||||
#include <common/wide_integer.h>
|
||||
|
||||
using Int8 = int8_t;
|
||||
using Int16 = int16_t;
|
||||
@ -21,106 +18,24 @@ using UInt16 = uint16_t;
|
||||
using UInt32 = uint32_t;
|
||||
using UInt64 = uint64_t;
|
||||
|
||||
using Int128 = __int128;
|
||||
using String = std::string;
|
||||
|
||||
using wInt256 = wide::integer<256, signed>;
|
||||
using wUInt256 = wide::integer<256, unsigned>;
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static_assert(sizeof(wInt256) == 32);
|
||||
static_assert(sizeof(wUInt256) == 32);
|
||||
using UInt8 = ::UInt8;
|
||||
using UInt16 = ::UInt16;
|
||||
using UInt32 = ::UInt32;
|
||||
using UInt64 = ::UInt64;
|
||||
|
||||
using Int8 = ::Int8;
|
||||
using Int16 = ::Int16;
|
||||
using Int32 = ::Int32;
|
||||
using Int64 = ::Int64;
|
||||
|
||||
using Float32 = float;
|
||||
using Float64 = double;
|
||||
|
||||
using String = std::string;
|
||||
|
||||
/// The standard library type traits, such as std::is_arithmetic, with one exception
|
||||
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
|
||||
/// So instead of using the std type_traits, we use our own version which allows extension.
|
||||
template <typename T>
|
||||
struct is_signed
|
||||
{
|
||||
static constexpr bool value = std::is_signed_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_signed<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_signed_v = is_signed<T>::value;
|
||||
|
||||
template <typename T>
|
||||
struct is_unsigned
|
||||
{
|
||||
static constexpr bool value = std::is_unsigned_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
|
||||
|
||||
|
||||
/// TODO: is_integral includes char, char8_t and wchar_t.
|
||||
template <typename T>
|
||||
struct is_integer
|
||||
{
|
||||
static constexpr bool value = std::is_integral_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_integer<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_integer_v = is_integer<T>::value;
|
||||
|
||||
|
||||
template <typename T>
|
||||
struct is_arithmetic
|
||||
{
|
||||
static constexpr bool value = std::is_arithmetic_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
|
||||
template <typename T>
|
||||
struct make_unsigned
|
||||
{
|
||||
typedef std::make_unsigned_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
|
||||
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
|
||||
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
|
||||
|
||||
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
|
||||
|
||||
template <typename T>
|
||||
struct make_signed
|
||||
{
|
||||
typedef std::make_signed_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_signed<wInt256> { using type = wInt256; };
|
||||
template <> struct make_signed<wUInt256> { using type = wInt256; };
|
||||
|
||||
template <typename T> using make_signed_t = typename make_signed<T>::type;
|
||||
|
||||
template <typename T>
|
||||
struct is_big_int
|
||||
{
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_big_int_v = is_big_int<T>::value;
|
||||
|
||||
template <typename To, typename From>
|
||||
inline To bigint_cast(const From & x [[maybe_unused]])
|
||||
{
|
||||
return static_cast<To>(x);
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
SET(VERSION_REVISION 54440)
|
||||
SET(VERSION_REVISION 54441)
|
||||
SET(VERSION_MAJOR 20)
|
||||
SET(VERSION_MINOR 10)
|
||||
SET(VERSION_PATCH 1)
|
||||
|
@ -28,7 +28,7 @@ elseif (COMPILER_CLANG)
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fchar8_t")
|
||||
endif ()
|
||||
else ()
|
||||
set (CLANG_MINIMUM_VERSION 8)
|
||||
set (CLANG_MINIMUM_VERSION 9)
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
|
||||
message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
|
||||
endif ()
|
||||
|
2
debian/rules
vendored
2
debian/rules
vendored
@ -18,7 +18,7 @@ ifeq ($(CCACHE_PREFIX),distcc)
|
||||
THREADS_COUNT=$(shell distcc -j)
|
||||
endif
|
||||
ifeq ($(THREADS_COUNT),)
|
||||
THREADS_COUNT=$(shell echo $$(( $$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu || echo 8) / 2 )) )
|
||||
THREADS_COUNT=$(shell nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu || echo 4)
|
||||
endif
|
||||
DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT)
|
||||
|
||||
|
@ -11,7 +11,7 @@ RUN apt-get update \
|
||||
&& echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
|
||||
&& apt-key add /tmp/llvm-snapshot.gpg.key \
|
||||
&& export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
|
||||
&& echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-11 main" >> \
|
||||
/etc/apt/sources.list
|
||||
|
||||
# initial packages
|
||||
@ -36,12 +36,11 @@ RUN apt-get update \
|
||||
clang-${LLVM_VERSION} \
|
||||
lld-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
clang-9 \
|
||||
lld-9 \
|
||||
clang-tidy-9 \
|
||||
clang-8 \
|
||||
lld-8 \
|
||||
clang-tidy-8 \
|
||||
clang-11 \
|
||||
clang-tidy-11 \
|
||||
lld-11 \
|
||||
llvm-11 \
|
||||
llvm-11-dev \
|
||||
libicu-dev \
|
||||
libreadline-dev \
|
||||
ninja-build \
|
||||
|
@ -18,7 +18,7 @@ ccache --zero-stats ||:
|
||||
ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
|
||||
rm -f CMakeCache.txt
|
||||
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS ..
|
||||
ninja -j $(($(nproc) / 2)) $NINJA_FLAGS clickhouse-bundle
|
||||
ninja $NINJA_FLAGS clickhouse-bundle
|
||||
mv ./programs/clickhouse* /output
|
||||
mv ./src/unit_tests_dbms /output
|
||||
find . -name '*.so' -print -exec mv '{}' /output \;
|
||||
|
@ -35,7 +35,7 @@ function download
|
||||
# wget -O- -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/performance/performance.tgz" \
|
||||
# | tar --strip-components=1 -zxv
|
||||
|
||||
wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-10_debug_none_bundled_unsplitted_disable_False_binary/clickhouse"
|
||||
wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-11_debug_none_bundled_unsplitted_disable_False_binary/clickhouse"
|
||||
chmod +x clickhouse
|
||||
ln -s ./clickhouse ./clickhouse-server
|
||||
ln -s ./clickhouse ./clickhouse-client
|
||||
@ -227,4 +227,4 @@ EOF
|
||||
;&
|
||||
esac
|
||||
|
||||
exit $task_exit_code
|
||||
exit $task_exit_code
|
||||
|
@ -1,4 +1,4 @@
|
||||
<yandex>
|
||||
<yandex>
|
||||
<http_port remove="remove"/>
|
||||
<mysql_port remove="remove"/>
|
||||
<interserver_http_port remove="remove"/>
|
||||
@ -22,4 +22,6 @@
|
||||
<uncompressed_cache_size>1000000000</uncompressed_cache_size>
|
||||
|
||||
<asynchronous_metrics_update_period_s>10</asynchronous_metrics_update_period_s>
|
||||
|
||||
<remap_executable replace="replace">true</remap_executable>
|
||||
</yandex>
|
||||
|
@ -45,6 +45,18 @@ Clusters are set like this:
|
||||
<remote_servers>
|
||||
<logs>
|
||||
<shard>
|
||||
<!-- Inter-server per-cluster secret for Distributed queries
|
||||
default: no secret (no authentication will be performed)
|
||||
|
||||
If set, then Distributed queries will be validated on shards, so at least:
|
||||
- such cluster should exist on the shard,
|
||||
- such cluster should have the same secret.
|
||||
|
||||
And also (and which is more important), the initial_user will
|
||||
be used as current user for the query.
|
||||
-->
|
||||
<!-- <secret></secret> -->
|
||||
|
||||
<!-- Optional. Shard weight when writing data. Default: 1. -->
|
||||
<weight>1</weight>
|
||||
<!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
|
||||
|
@ -6,11 +6,11 @@ toc_title: Playground
|
||||
# ClickHouse Playground {#clickhouse-playground}
|
||||
|
||||
[ClickHouse Playground](https://play.clickhouse.tech) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster.
|
||||
Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. There’s also a selection of ClickHouse LTS releases to experiment with.
|
||||
Several example datasets are available in Playground as well as sample queries that show ClickHouse features. There’s also a selection of ClickHouse LTS releases to experiment with.
|
||||
|
||||
ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance (4 vCPU, 32 GB RAM) hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md).
|
||||
|
||||
You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md).
|
||||
You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md).
|
||||
|
||||
## Credentials {#credentials}
|
||||
|
||||
@ -60,7 +60,7 @@ clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground
|
||||
## Implementation Details {#implementation-details}
|
||||
|
||||
ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md).
|
||||
The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add extra layer of protection and improved global connectivity.
|
||||
The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add an extra layer of protection and improved global connectivity.
|
||||
|
||||
!!! warning "Warning"
|
||||
Exposing ClickHouse server to public internet in any other situation is **strongly not recommended**. Make sure it listens only on private network and is covered by properly configured firewall.
|
||||
Exposing the ClickHouse server to the public internet in any other situation is **strongly not recommended**. Make sure it listens only on a private network and is covered by a properly configured firewall.
|
||||
|
@ -940,6 +940,8 @@ This algorithm chooses the first replica in the set or a random replica if the f
|
||||
|
||||
The `first_or_random` algorithm solves the problem of the `in_order` algorithm. With `in_order`, if one replica goes down, the next one gets a double load while the remaining replicas handle the usual amount of traffic. When using the `first_or_random` algorithm, the load is evenly distributed among replicas that are still available.
|
||||
|
||||
It's possible to explicitly define what the first replica is by using the setting `load_balancing_first_offset`. This gives more control to rebalance query workloads among replicas.
|
||||
|
||||
### Round Robin {#load_balancing-round_robin}
|
||||
|
||||
``` sql
|
||||
|
@ -6,7 +6,7 @@ toc_title: ANSI Compatibility
|
||||
# ANSI SQL Compatibility of ClickHouse SQL Dialect {#ansi-sql-compatibility-of-clickhouse-sql-dialect}
|
||||
|
||||
!!! note "Note"
|
||||
This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013.
|
||||
This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of [ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8).
|
||||
|
||||
## Differences in Behaviour {#differences-in-behaviour}
|
||||
|
||||
@ -77,6 +77,16 @@ The following table lists cases when query feature works in ClickHouse, but beha
|
||||
| E071-05 | Columns combined via table operators need not have exactly the same data type | Yes{.text-success} | |
|
||||
| E071-06 | Table operators in subqueries | Yes{.text-success} | |
|
||||
| **E081** | **Basic privileges** | **Partial**{.text-warning} | Work in progress |
|
||||
| E081-01 | SELECT privilege at the table level | | |
|
||||
| E081-02 | DELETE privilege | | |
|
||||
| E081-03 | INSERT privilege at the table level | | |
|
||||
| E081-04 | UPDATE privilege at the table level | | |
|
||||
| E081-05 | UPDATE privilege at the column level | | |
|
||||
| E081-06 | REFERENCES privilege at the table level | | |
|
||||
| E081-07 | REFERENCES privilege at the column level | | |
|
||||
| E081-08 | WITH GRANT OPTION | | |
|
||||
| E081-09 | USAGE privilege | | |
|
||||
| E081-10 | EXECUTE privilege | | |
|
||||
| **E091** | **Set functions** | **Yes**{.text-success} | |
|
||||
| E091-01 | AVG | Yes{.text-success} | |
|
||||
| E091-02 | COUNT | Yes{.text-success} | |
|
||||
@ -169,6 +179,7 @@ The following table lists cases when query feature works in ClickHouse, but beha
|
||||
| **F471** | **Scalar subquery values** | **Yes**{.text-success} | |
|
||||
| **F481** | **Expanded NULL predicate** | **Yes**{.text-success} | |
|
||||
| **F812** | **Basic flagging** | **No**{.text-danger} | |
|
||||
| **S011** | **Distinct data types** | | |
|
||||
| **T321** | **Basic SQL-invoked routines** | **No**{.text-danger} | |
|
||||
| T321-01 | User-defined functions with no overloading | No{.text-danger} | |
|
||||
| T321-02 | User-defined stored procedures with no overloading | No{.text-danger} | |
|
||||
|
@ -246,7 +246,7 @@ Installing unixODBC and the ODBC driver for PostgreSQL:
|
||||
$ sudo apt-get install -y unixodbc odbcinst odbc-postgresql
|
||||
```
|
||||
|
||||
Configuring `/etc/odbc.ini` (or `~/.odbc.ini`):
|
||||
Configuring `/etc/odbc.ini` (or `~/.odbc.ini` if you signed in under a user that runs ClickHouse):
|
||||
|
||||
``` text
|
||||
[DEFAULT]
|
||||
@ -321,7 +321,7 @@ You may need to edit `odbc.ini` to specify the full path to the library with the
|
||||
|
||||
Ubuntu OS.
|
||||
|
||||
Installing the driver: :
|
||||
Installing the ODBC driver for connecting to MS SQL:
|
||||
|
||||
``` bash
|
||||
$ sudo apt-get install tdsodbc freetds-bin sqsh
|
||||
@ -329,7 +329,7 @@ $ sudo apt-get install tdsodbc freetds-bin sqsh
|
||||
|
||||
Configuring the driver:
|
||||
|
||||
``` bash
|
||||
```bash
|
||||
$ cat /etc/freetds/freetds.conf
|
||||
...
|
||||
|
||||
@ -339,8 +339,11 @@ Configuring the driver:
|
||||
tds version = 7.0
|
||||
client charset = UTF-8
|
||||
|
||||
# test TDS connection
|
||||
$ sqsh -S MSSQL -D database -U user -P password
|
||||
|
||||
|
||||
$ cat /etc/odbcinst.ini
|
||||
...
|
||||
|
||||
[FreeTDS]
|
||||
Description = FreeTDS
|
||||
@ -349,8 +352,8 @@ Configuring the driver:
|
||||
FileUsage = 1
|
||||
UsageCount = 5
|
||||
|
||||
$ cat ~/.odbc.ini
|
||||
...
|
||||
$ cat /etc/odbc.ini
|
||||
# $ cat ~/.odbc.ini # if you signed in under a user that runs ClickHouse
|
||||
|
||||
[MSSQL]
|
||||
Description = FreeTDS
|
||||
@ -360,8 +363,15 @@ Configuring the driver:
|
||||
UID = test
|
||||
PWD = test
|
||||
Port = 1433
|
||||
|
||||
|
||||
# (optional) test ODBC connection (to use isql-tool install the [unixodbc](https://packages.debian.org/sid/unixodbc)-package)
|
||||
$ isql -v MSSQL "user" "password"
|
||||
```
|
||||
|
||||
Remarks:
|
||||
- to determine the earliest TDS version that is supported by a particular SQL Server version, refer to the product documentation or look at [MS-TDS Product Behavior](https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/135d0ebe-5c4c-4a94-99bf-1811eccb9f4a)
|
||||
|
||||
Configuring the dictionary in ClickHouse:
|
||||
|
||||
``` xml
|
||||
|
@ -111,7 +111,7 @@ dictHas('dict_name', id_expr)
|
||||
**Parameters**
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -189,8 +189,8 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr)
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `attr_name` — Name of the column of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
|
||||
- `default_value_expr` — Value which is returned if the dictionary doesn’t contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a value in the data type configured for the `attr_name` attribute.
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
- `default_value_expr` — Value returned if the dictionary doesn’t contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning the value in the data type configured for the `attr_name` attribute.
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
@ -1,38 +1,59 @@
|
||||
# ClickHouse Playground {#clickhouse-playground}
|
||||
|
||||
ClickHouse Playground позволяет моментально выполнить запросы к ClickHouse из бразуера.
|
||||
В Playground доступны несколько тестовых массивов данных и примеры запросов, которые показывают некоторые отличительные черты ClickHouse.
|
||||
[ClickHouse Playground](https://play.clickhouse.tech) позволяет пользователям экспериментировать с ClickHouse, мгновенно выполняя запросы без настройки своего сервера или кластера.
|
||||
В Playground доступны несколько тестовых массивов данных, а также примеры запросов, которые показывают возможности ClickHouse. Кроме того, вы можете выбрать LTS релиз ClickHouse, который хотите протестировать.
|
||||
|
||||
Запросы выполняются под пользователем с правами `readonly` для которого есть следующие ограничения:
|
||||
ClickHouse Playground дает возможность поработать с [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) в конфигурации m2.small (4 vCPU, 32 ГБ ОЗУ), которую предосталяет [Яндекс.Облако](https://cloud.yandex.com/). Дополнительную информацию об облачных провайдерах читайте в разделе [Поставщики облачных услуг ClickHouse](../commercial/cloud.md).
|
||||
|
||||
Вы можете отправлять запросы к Playground с помощью любого HTTP-клиента, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), также можно установить соединение с помощью драйверов [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Более подробная информация о программных продуктах, поддерживающих ClickHouse, доступна [здесь](../interfaces/index.md).
|
||||
|
||||
## Параметры доступа {#credentials}
|
||||
|
||||
| Параметр | Значение |
|
||||
|:--------------------|:----------------------------------------|
|
||||
| Конечная точка HTTPS| `https://play-api.clickhouse.tech:8443` |
|
||||
| Конечная точка TCP | `play-api.clickhouse.tech:9440` |
|
||||
| Пользователь | `playground` |
|
||||
| Пароль | `clickhouse` |
|
||||
|
||||
Также можно подключаться к ClickHouse определённых релизов, чтобы протестировать их различия (порты и пользователь / пароль остаются неизменными):
|
||||
|
||||
- 20.3 LTS: `play-api-v20-3.clickhouse.tech`
|
||||
- 19.14 LTS: `play-api-v19-14.clickhouse.tech`
|
||||
|
||||
!!! note "Примечание"
|
||||
Для всех этих конечных точек требуется безопасное соединение TLS.
|
||||
|
||||
## Ограничения {#limitations}
|
||||
|
||||
Запросы выполняются под пользователем с правами `readonly`, для которого есть следующие ограничения:
|
||||
- запрещены DDL запросы
|
||||
- запрещены INSERT запросы
|
||||
|
||||
Также установлены следующие опции:
|
||||
- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes)
|
||||
- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows)
|
||||
- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode)
|
||||
- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time)
|
||||
- [max\_result\_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes)
|
||||
- [max\_result\_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows)
|
||||
- [result\_overflow\_mode=break](../operations/settings/query_complexity/#result-overflow-mode)
|
||||
- [max\_execution\_time=60000](../operations/settings/query_complexity/#max-execution-time)
|
||||
|
||||
ClickHouse Playground соответствует конфигурации m2.small хосту
|
||||
[Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse)
|
||||
запущеному в [Яндекс.Облаке](https://cloud.yandex.com/).
|
||||
Больше информации про [облачных провайдерах](../commercial/cloud.md).
|
||||
## Примеры {#examples}
|
||||
|
||||
Веб интерфейс ClickHouse Playground делает запросы через ClickHouse HTTP API.
|
||||
Бекендом служит обычный кластер ClickHouse.
|
||||
ClickHouse HTTP интерфейс также доступен как часть Playground.
|
||||
|
||||
Запросы к Playground могут быть выполнены с помощью curl/wget, а также через соединеие JDBC/ODBC драйвера
|
||||
Больше информации про приложения с поддержкой ClickHouse доступно в разделе [Интерфейсы](../interfaces/index.md).
|
||||
|
||||
| Параметр | Значение |
|
||||
|:-----------------|:--------------------------------------|
|
||||
| Адрес | https://play-api.clickhouse.tech:8443 |
|
||||
| Имя пользователя | `playground` |
|
||||
| Пароль | `clickhouse` |
|
||||
|
||||
Требуется SSL соединение.
|
||||
Пример конечной точки HTTPS с `curl`:
|
||||
|
||||
``` bash
|
||||
curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets"
|
||||
curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets"
|
||||
```
|
||||
|
||||
Пример конечной точки TCP с [CLI](../interfaces/cli.md):
|
||||
|
||||
``` bash
|
||||
clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'"
|
||||
```
|
||||
|
||||
## Детали реализации {#implementation-details}
|
||||
|
||||
Веб-интерфейс ClickHouse Playground выполняет запросы через ClickHouse [HTTP API](../interfaces/http.md).
|
||||
Бэкэнд Playground - это кластер ClickHouse без дополнительных серверных приложений. Как упоминалось выше, способы подключения по HTTPS и TCP/TLS общедоступны как часть Playground. Они проксируются через [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) для добавления дополнительного уровня защиты и улучшенного глобального подключения.
|
||||
|
||||
!!! warning "Предупреждение"
|
||||
Открывать сервер ClickHouse для публичного доступа в любой другой ситуации **настоятельно не рекомендуется**. Убедитесь, что он настроен только на частную сеть и защищен брандмауэром.
|
||||
|
@ -103,7 +103,7 @@ dictHas('dict_name', id)
|
||||
**Параметры**
|
||||
|
||||
- `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal).
|
||||
- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md).
|
||||
- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md) или [Tuple](../../sql-reference/functions/ext-dict-functions.md) в зависимости от конфигурации словаря.
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
@ -179,7 +179,7 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr)
|
||||
|
||||
- `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal).
|
||||
- `attr_name` — имя столбца словаря. [Строковый литерал](../syntax.md#syntax-string-literal).
|
||||
- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md).
|
||||
- `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md) или [Tuple](../../sql-reference/functions/ext-dict-functions.md) в зависимости от конфигурации словаря.
|
||||
- `default_value_expr` — значение, возвращаемое в том случае, когда словарь не содержит строки с заданным ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращающее значение с типом данных, сконфигурированным для атрибута `attr_name`.
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
@ -85,7 +85,12 @@ public:
|
||||
std::string cur_host = i >= hosts_.size() ? "localhost" : hosts_[i];
|
||||
|
||||
connections.emplace_back(std::make_unique<ConnectionPool>(
|
||||
concurrency, cur_host, cur_port, default_database_, user_, password_, "benchmark", Protocol::Compression::Enable, secure));
|
||||
concurrency,
|
||||
cur_host, cur_port,
|
||||
default_database_, user_, password_,
|
||||
"", /* cluster */
|
||||
"", /* cluster_secret */
|
||||
"benchmark", Protocol::Compression::Enable, secure));
|
||||
comparison_info_per_interval.emplace_back(std::make_shared<Stats>());
|
||||
comparison_info_total.emplace_back(std::make_shared<Stats>());
|
||||
}
|
||||
|
@ -701,6 +701,8 @@ private:
|
||||
connection_parameters.default_database,
|
||||
connection_parameters.user,
|
||||
connection_parameters.password,
|
||||
"", /* cluster */
|
||||
"", /* cluster_secret */
|
||||
"client",
|
||||
connection_parameters.compression,
|
||||
connection_parameters.security);
|
||||
|
@ -26,6 +26,8 @@ void Suggest::load(const ConnectionParameters & connection_parameters, size_t su
|
||||
connection_parameters.default_database,
|
||||
connection_parameters.user,
|
||||
connection_parameters.password,
|
||||
"" /* cluster */,
|
||||
"" /* cluster_secret */,
|
||||
"client",
|
||||
connection_parameters.compression,
|
||||
connection_parameters.security);
|
||||
|
@ -32,6 +32,7 @@
|
||||
#include <Common/getExecutablePath.h>
|
||||
#include <Common/ThreadProfileEvents.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/remapExecutable.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/UseSSL.h>
|
||||
#include <Interpreters/AsynchronousMetrics.h>
|
||||
@ -305,6 +306,13 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
/// After full config loaded
|
||||
{
|
||||
if (config().getBool("remap_executable", false))
|
||||
{
|
||||
LOG_DEBUG(log, "Will remap executable in memory.");
|
||||
remapExecutable();
|
||||
LOG_DEBUG(log, "The code in memory has been successfully remapped.");
|
||||
}
|
||||
|
||||
if (config().getBool("mlock_executable", false))
|
||||
{
|
||||
if (hasLinuxCapability(CAP_IPC_LOCK))
|
||||
@ -530,6 +538,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
if (config->has("max_partition_size_to_drop"))
|
||||
global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop"));
|
||||
|
||||
if (config->has("zookeeper"))
|
||||
global_context->reloadZooKeeperIfChanged(config);
|
||||
|
||||
global_context->updateStorageConfiguration(*config);
|
||||
},
|
||||
/* already_loaded = */ true);
|
||||
|
@ -302,12 +302,37 @@
|
||||
-->
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<!-- Reallocate memory for machine code ("text") using huge pages. Highly experimental. -->
|
||||
<remap_executable>false</remap_executable>
|
||||
|
||||
<!-- Configuration of clusters that could be used in Distributed tables.
|
||||
https://clickhouse.tech/docs/en/operations/table_engines/distributed/
|
||||
-->
|
||||
<remote_servers incl="clickhouse_remote_servers" >
|
||||
<!-- Test only shard config for testing distributed storage -->
|
||||
<test_shard_localhost>
|
||||
<!-- Inter-server per-cluster secret for Distributed queries
|
||||
default: no secret (no authentication will be performed)
|
||||
|
||||
If set, then Distributed queries will be validated on shards, so at least:
|
||||
- such cluster should exist on the shard,
|
||||
- such cluster should have the same secret.
|
||||
|
||||
And also (and which is more important), the initial_user will
|
||||
be used as current user for the query.
|
||||
|
||||
Right now the protocol is pretty simple and it only takes into account:
|
||||
- cluster name
|
||||
- query
|
||||
|
||||
Also it will be nice if the following will be implemented:
|
||||
- source hostname (see interserver_http_host), but then it will depends from DNS,
|
||||
it can use IP address instead, but then the you need to get correct on the initiator node.
|
||||
- target hostname / ip address (same notes as for source hostname)
|
||||
- time-based security tokens
|
||||
-->
|
||||
<!-- <secret></secret> -->
|
||||
|
||||
<shard>
|
||||
<!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
|
||||
<!-- <internal_replication>false</internal_replication> -->
|
||||
|
@ -186,7 +186,7 @@ void AccessControlManager::addUsersConfigStorage(
|
||||
{
|
||||
if (auto users_config_storage = typeid_cast<std::shared_ptr<UsersConfigAccessStorage>>(storage))
|
||||
{
|
||||
if (users_config_storage->getStoragePath() == users_config_path_)
|
||||
if (users_config_storage->isPathEqual(users_config_path_))
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -229,7 +229,7 @@ void AccessControlManager::addDiskStorage(const String & storage_name_, const St
|
||||
{
|
||||
if (auto disk_storage = typeid_cast<std::shared_ptr<DiskAccessStorage>>(storage))
|
||||
{
|
||||
if (disk_storage->isStoragePathEqual(directory_))
|
||||
if (disk_storage->isPathEqual(directory_))
|
||||
{
|
||||
if (readonly_)
|
||||
disk_storage->setReadOnly(readonly_);
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/AccessType.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <ext/range.h>
|
||||
#include <ext/push_back.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Access/AccessRightsElement.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
@ -1,13 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <boost/algorithm/string/case_conv.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using Strings = std::vector<String>;
|
||||
|
||||
/// Represents an access type which can be granted on databases, tables, columns, etc.
|
||||
enum class AccessType
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
@ -11,6 +11,9 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using Strings = std::vector<String>;
|
||||
|
||||
/// Represents lists of hosts an user is allowed to connect to server from.
|
||||
class AllowedClientHosts
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Poco/SHA1Engine.h>
|
||||
|
@ -33,6 +33,9 @@
|
||||
#include <Interpreters/InterpreterShowGrantsQuery.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <Poco/JSON/JSON.h>
|
||||
#include <Poco/JSON/Object.h>
|
||||
#include <Poco/JSON/Stringifier.h>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <boost/range/algorithm_ext/push_back.hpp>
|
||||
@ -342,9 +345,22 @@ DiskAccessStorage::~DiskAccessStorage()
|
||||
}
|
||||
|
||||
|
||||
bool DiskAccessStorage::isStoragePathEqual(const String & directory_path_) const
|
||||
String DiskAccessStorage::getStorageParamsJSON() const
|
||||
{
|
||||
return getStoragePath() == makeDirectoryPathCanonical(directory_path_);
|
||||
std::lock_guard lock{mutex};
|
||||
Poco::JSON::Object json;
|
||||
json.set("path", directory_path);
|
||||
if (readonly)
|
||||
json.set("readonly", readonly.load());
|
||||
std::ostringstream oss;
|
||||
Poco::JSON::Stringifier::stringify(json, oss);
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
|
||||
bool DiskAccessStorage::isPathEqual(const String & directory_path_) const
|
||||
{
|
||||
return getPath() == makeDirectoryPathCanonical(directory_path_);
|
||||
}
|
||||
|
||||
|
||||
|
@ -18,12 +18,13 @@ public:
|
||||
~DiskAccessStorage() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
String getStorageParamsJSON() const override;
|
||||
|
||||
String getStoragePath() const override { return directory_path; }
|
||||
bool isStoragePathEqual(const String & directory_path_) const;
|
||||
String getPath() const { return directory_path; }
|
||||
bool isPathEqual(const String & directory_path_) const;
|
||||
|
||||
void setReadOnly(bool readonly_) { readonly = readonly_; }
|
||||
bool isStorageReadOnly() const override { return readonly; }
|
||||
bool isReadOnly() const { return readonly; }
|
||||
|
||||
private:
|
||||
std::optional<UUID> findImpl(EntityType type, const String & name) const override;
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/RowPolicy.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <boost/smart_ptr/atomic_shared_ptr.hpp>
|
||||
#include <unordered_map>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <Access/SettingsConstraints.h>
|
||||
#include <Access/SettingsProfileElement.h>
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/LDAPParams.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/IAccessEntity.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <functional>
|
||||
@ -25,8 +25,9 @@ public:
|
||||
/// Returns the name of this storage.
|
||||
const String & getStorageName() const { return storage_name; }
|
||||
virtual const char * getStorageType() const = 0;
|
||||
virtual String getStoragePath() const { return {}; }
|
||||
virtual bool isStorageReadOnly() const { return false; }
|
||||
|
||||
/// Returns a JSON with the parameters of the storage. It's up to the storage type to fill the JSON.
|
||||
virtual String getStorageParamsJSON() const { return "{}"; }
|
||||
|
||||
using EntityType = IAccessEntity::Type;
|
||||
using EntityTypeInfo = IAccessEntity::TypeInfo;
|
||||
|
@ -5,7 +5,7 @@
|
||||
#endif
|
||||
|
||||
#include <Access/LDAPParams.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#if USE_LDAP
|
||||
# include <ldap.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include <chrono>
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Access/EnabledSettings.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
|
@ -10,6 +10,9 @@
|
||||
#include <Core/Settings.h>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <Poco/MD5Engine.h>
|
||||
#include <Poco/JSON/JSON.h>
|
||||
#include <Poco/JSON/Object.h>
|
||||
#include <Poco/JSON/Stringifier.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
@ -482,12 +485,29 @@ UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_,
|
||||
UsersConfigAccessStorage::~UsersConfigAccessStorage() = default;
|
||||
|
||||
|
||||
String UsersConfigAccessStorage::getStoragePath() const
|
||||
String UsersConfigAccessStorage::getStorageParamsJSON() const
|
||||
{
|
||||
std::lock_guard lock{load_mutex};
|
||||
Poco::JSON::Object json;
|
||||
if (!path.empty())
|
||||
json.set("path", path);
|
||||
std::ostringstream oss;
|
||||
Poco::JSON::Stringifier::stringify(json, oss);
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
|
||||
String UsersConfigAccessStorage::getPath() const
|
||||
{
|
||||
std::lock_guard lock{load_mutex};
|
||||
return path;
|
||||
}
|
||||
|
||||
bool UsersConfigAccessStorage::isPathEqual(const String & path_) const
|
||||
{
|
||||
return getPath() == path_;
|
||||
}
|
||||
|
||||
|
||||
void UsersConfigAccessStorage::setConfig(const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
|
@ -26,8 +26,10 @@ public:
|
||||
~UsersConfigAccessStorage() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
String getStoragePath() const override;
|
||||
bool isStorageReadOnly() const override { return true; }
|
||||
String getStorageParamsJSON() const override;
|
||||
|
||||
String getPath() const;
|
||||
bool isPathEqual(const String & path_) const;
|
||||
|
||||
void setConfig(const Poco::Util::AbstractConfiguration & config);
|
||||
|
||||
|
@ -6,7 +6,7 @@
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
@ -5,7 +5,7 @@
|
||||
#include <vector>
|
||||
#include <type_traits>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/ColumnNumbers.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Common/Exception.h>
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
#include <IO/VarInt.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
|
@ -117,6 +117,10 @@ endif ()
|
||||
|
||||
add_library(clickhouse_common_io ${clickhouse_common_io_headers} ${clickhouse_common_io_sources})
|
||||
|
||||
if (SPLIT_SHARED_LIBRARIES)
|
||||
target_compile_definitions(clickhouse_common_io PRIVATE SPLIT_SHARED_LIBRARIES)
|
||||
endif ()
|
||||
|
||||
add_library (clickhouse_malloc OBJECT Common/malloc.cpp)
|
||||
set_source_files_properties(Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-builtin")
|
||||
|
||||
|
@ -17,12 +17,15 @@
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/DNSResolver.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Compression/CompressionFactory.h>
|
||||
#include <Processors/Pipe.h>
|
||||
#include <Processors/ISink.h>
|
||||
#include <Processors/Executors/PipelineExecutor.h>
|
||||
#include <Processors/ConcatProcessor.h>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config_version.h>
|
||||
@ -171,8 +174,26 @@ void Connection::sendHello()
|
||||
// NOTE For backward compatibility of the protocol, client cannot send its version_patch.
|
||||
writeVarUInt(client_revision, *out);
|
||||
writeStringBinary(default_database, *out);
|
||||
writeStringBinary(user, *out);
|
||||
writeStringBinary(password, *out);
|
||||
/// If interserver-secret is used, one do not need password
|
||||
/// (NOTE we do not check for DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET, since we cannot ignore inter-server secret if it was requested)
|
||||
if (!cluster_secret.empty())
|
||||
{
|
||||
writeStringBinary(USER_INTERSERVER_MARKER, *out);
|
||||
writeStringBinary("" /* password */, *out);
|
||||
|
||||
#if USE_SSL
|
||||
sendClusterNameAndSalt();
|
||||
#else
|
||||
throw Exception(
|
||||
"Inter-server secret support is disabled, because ClickHouse was built without SSL library",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED);
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
writeStringBinary(user, *out);
|
||||
writeStringBinary(password, *out);
|
||||
}
|
||||
|
||||
out->next();
|
||||
}
|
||||
@ -288,6 +309,19 @@ void Connection::forceConnected(const ConnectionTimeouts & timeouts)
|
||||
}
|
||||
}
|
||||
|
||||
#if USE_SSL
|
||||
void Connection::sendClusterNameAndSalt()
|
||||
{
|
||||
pcg64_fast rng(randomSeed());
|
||||
UInt64 rand = rng();
|
||||
|
||||
salt = encodeSHA256(&rand, sizeof(rand));
|
||||
|
||||
writeStringBinary(cluster, *out);
|
||||
writeStringBinary(salt, *out);
|
||||
}
|
||||
#endif
|
||||
|
||||
bool Connection::ping()
|
||||
{
|
||||
// LOG_TRACE(log_wrapper.get(), "Ping");
|
||||
@ -406,6 +440,37 @@ void Connection::sendQuery(
|
||||
else
|
||||
writeStringBinary("" /* empty string is a marker of the end of settings */, *out);
|
||||
|
||||
/// Interserver secret
|
||||
if (server_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET)
|
||||
{
|
||||
/// Hash
|
||||
///
|
||||
/// Send correct hash only for !INITIAL_QUERY, due to:
|
||||
/// - this will avoid extra protocol complexity for simplest cases
|
||||
/// - there is no need in hash for the INITIAL_QUERY anyway
|
||||
/// (since there is no secure/unsecure changes)
|
||||
if (client_info && !cluster_secret.empty() && client_info->query_kind != ClientInfo::QueryKind::INITIAL_QUERY)
|
||||
{
|
||||
#if USE_SSL
|
||||
std::string data(salt);
|
||||
data += cluster_secret;
|
||||
data += query;
|
||||
data += query_id;
|
||||
data += client_info->initial_user;
|
||||
/// TODO: add source/target host/ip-address
|
||||
|
||||
std::string hash = encodeSHA256(data);
|
||||
writeStringBinary(hash, *out);
|
||||
#else
|
||||
throw Exception(
|
||||
"Inter-server secret support is disabled, because ClickHouse was built without SSL library",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED);
|
||||
#endif
|
||||
}
|
||||
else
|
||||
writeStringBinary("", *out);
|
||||
}
|
||||
|
||||
writeVarUInt(stage, *out);
|
||||
writeVarUInt(static_cast<bool>(compression), *out);
|
||||
|
||||
|
@ -83,6 +83,8 @@ public:
|
||||
Connection(const String & host_, UInt16 port_,
|
||||
const String & default_database_,
|
||||
const String & user_, const String & password_,
|
||||
const String & cluster_,
|
||||
const String & cluster_secret_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
@ -90,6 +92,8 @@ public:
|
||||
:
|
||||
host(host_), port(port_), default_database(default_database_),
|
||||
user(user_), password(password_),
|
||||
cluster(cluster_),
|
||||
cluster_secret(cluster_secret_),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
secure(secure_),
|
||||
@ -191,6 +195,11 @@ private:
|
||||
String user;
|
||||
String password;
|
||||
|
||||
/// For inter-server authorization
|
||||
String cluster;
|
||||
String cluster_secret;
|
||||
String salt;
|
||||
|
||||
/// Address is resolved during the first connection (or the following reconnects)
|
||||
/// Use it only for logging purposes
|
||||
std::optional<Poco::Net::SocketAddress> current_resolved_address;
|
||||
@ -269,6 +278,10 @@ private:
|
||||
void connect(const ConnectionTimeouts & timeouts);
|
||||
void sendHello();
|
||||
void receiveHello();
|
||||
|
||||
#if USE_SSL
|
||||
void sendClusterNameAndSalt();
|
||||
#endif
|
||||
bool ping();
|
||||
|
||||
Block receiveData();
|
||||
|
@ -54,6 +54,8 @@ public:
|
||||
const String & default_database_,
|
||||
const String & user_,
|
||||
const String & password_,
|
||||
const String & cluster_,
|
||||
const String & cluster_secret_,
|
||||
const String & client_name_ = "client",
|
||||
Protocol::Compression compression_ = Protocol::Compression::Enable,
|
||||
Protocol::Secure secure_ = Protocol::Secure::Disable,
|
||||
@ -65,6 +67,8 @@ public:
|
||||
default_database(default_database_),
|
||||
user(user_),
|
||||
password(password_),
|
||||
cluster(cluster_),
|
||||
cluster_secret(cluster_secret_),
|
||||
client_name(client_name_),
|
||||
compression(compression_),
|
||||
secure(secure_),
|
||||
@ -109,6 +113,7 @@ protected:
|
||||
return std::make_shared<Connection>(
|
||||
host, port,
|
||||
default_database, user, password,
|
||||
cluster, cluster_secret,
|
||||
client_name, compression, secure);
|
||||
}
|
||||
|
||||
@ -119,6 +124,10 @@ private:
|
||||
String user;
|
||||
String password;
|
||||
|
||||
/// For inter-server authorization
|
||||
String cluster;
|
||||
String cluster_secret;
|
||||
|
||||
String client_name;
|
||||
Protocol::Compression compression; /// Whether to compress data when interacting with the server.
|
||||
Protocol::Secure secure; /// Whether to encrypt data when interacting with the server.
|
||||
|
@ -56,6 +56,9 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts
|
||||
return tryGetEntry(pool, timeouts, fail_message, settings);
|
||||
};
|
||||
|
||||
size_t offset = 0;
|
||||
if (settings)
|
||||
offset = settings->load_balancing_first_offset % nested_pools.size();
|
||||
GetPriorityFunc get_priority;
|
||||
switch (settings ? LoadBalancing(settings->load_balancing) : default_load_balancing)
|
||||
{
|
||||
@ -68,7 +71,7 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts
|
||||
case LoadBalancing::RANDOM:
|
||||
break;
|
||||
case LoadBalancing::FIRST_OR_RANDOM:
|
||||
get_priority = [](size_t i) -> size_t { return i >= 1; };
|
||||
get_priority = [offset](size_t i) -> size_t { return i != offset; };
|
||||
break;
|
||||
case LoadBalancing::ROUND_ROBIN:
|
||||
if (last_used >= nested_pools.size())
|
||||
@ -190,6 +193,9 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
|
||||
else
|
||||
throw DB::Exception("Unknown pool allocation mode", DB::ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
size_t offset = 0;
|
||||
if (settings)
|
||||
offset = settings->load_balancing_first_offset % nested_pools.size();
|
||||
GetPriorityFunc get_priority;
|
||||
switch (settings ? LoadBalancing(settings->load_balancing) : default_load_balancing)
|
||||
{
|
||||
@ -202,7 +208,7 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
|
||||
case LoadBalancing::RANDOM:
|
||||
break;
|
||||
case LoadBalancing::FIRST_OR_RANDOM:
|
||||
get_priority = [](size_t i) -> size_t { return i >= 1; };
|
||||
get_priority = [offset](size_t i) -> size_t { return i != offset; };
|
||||
break;
|
||||
case LoadBalancing::ROUND_ROBIN:
|
||||
if (last_used >= nested_pools.size())
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
|
||||
|
||||
|
@ -12,7 +12,7 @@
|
||||
#endif
|
||||
|
||||
#include <ext/bit_cast.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Columns/ColumnsCommon.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
namespace Poco::Util
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
#include <cpuid.h>
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <atomic>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
/** Allows to count number of simultaneously happening processes or current value of some metric.
|
||||
* - for high-level profiling.
|
||||
|
@ -3,7 +3,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Core/Names.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <Poco/Net/DNS.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
|
@ -2,7 +2,7 @@
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
#include <memory>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/Names.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <common/logger_useful.h>
|
||||
|
@ -3,7 +3,7 @@
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
#include <ostream>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/BigInt.h>
|
||||
#include <Common/UInt128.h>
|
||||
#include <common/unaligned.h>
|
||||
|
@ -9,7 +9,7 @@
|
||||
#include <boost/noncopyable.hpp>
|
||||
|
||||
#include <Core/Defines.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <IO/WriteBuffer.h>
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/NamePrompter.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Poco/String.h>
|
||||
|
||||
#include <unordered_map>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/Names.h>
|
||||
#include <Interpreters/StorageID.h>
|
||||
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
#include <common/types.h>
|
||||
#include <common/extended_types.h>
|
||||
|
||||
|
||||
/// To be sure, that this function is zero-cost for non-floating point types.
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/PODArray.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
@ -12,11 +12,26 @@ namespace DB
|
||||
{
|
||||
#pragma GCC diagnostic warning "-Wold-style-cast"
|
||||
|
||||
std::string encodeSHA256(const std::string_view & text)
|
||||
{
|
||||
return encodeSHA256(text.data(), text.size());
|
||||
}
|
||||
std::string encodeSHA256(const void * text, size_t size)
|
||||
{
|
||||
std::string out;
|
||||
out.resize(32);
|
||||
encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
|
||||
return out;
|
||||
}
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out)
|
||||
{
|
||||
encodeSHA256(text.data(), text.size(), out);
|
||||
}
|
||||
void encodeSHA256(const void * text, size_t size, unsigned char * out)
|
||||
{
|
||||
SHA256_CTX ctx;
|
||||
SHA256_Init(&ctx);
|
||||
SHA256_Update(&ctx, reinterpret_cast<const UInt8 *>(text.data()), text.size());
|
||||
SHA256_Update(&ctx, reinterpret_cast<const UInt8 *>(text), size);
|
||||
SHA256_Final(out, &ctx);
|
||||
}
|
||||
|
||||
|
@ -5,13 +5,18 @@
|
||||
#endif
|
||||
|
||||
#if USE_SSL
|
||||
# include <Core/Types.h>
|
||||
# include <common/types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/// Encodes `text` and puts the result to `out` which must be at least 32 bytes long.
|
||||
|
||||
/// Encodes `text` and returns it.
|
||||
std::string encodeSHA256(const std::string_view & text);
|
||||
std::string encodeSHA256(const void * text, size_t size);
|
||||
/// `out` must be at least 32 bytes long.
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out);
|
||||
void encodeSHA256(const void * text, size_t size, unsigned char * out);
|
||||
|
||||
/// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.
|
||||
String getOpenSSLErrors();
|
||||
|
@ -7,7 +7,6 @@
|
||||
#include <functional>
|
||||
#include <common/types.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Common/PoolBase.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/NetException.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <list>
|
||||
|
@ -13,7 +13,7 @@
|
||||
#include <type_traits>
|
||||
|
||||
#include <ext/bit_cast.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/extended_types.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
|
||||
|
@ -4,7 +4,8 @@
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <atomic>
|
||||
#include <Core/Types.h>
|
||||
#include <vector>
|
||||
#include <common/types.h>
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
#include "TaskStatsInfoGetter.h"
|
||||
#include <Common/Exception.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
|
||||
struct taskstats;
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/resource.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
#include <Poco/UTF8Encoding.h>
|
||||
|
||||
|
@ -3,7 +3,7 @@
|
||||
#include <cstring>
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#define UNICODE_BAR_CHAR_SIZE (strlen("█"))
|
||||
|
||||
|
@ -66,7 +66,8 @@ class Visitor<>
|
||||
public:
|
||||
using List = TypeList<>;
|
||||
|
||||
virtual ~Visitor() = default;
|
||||
protected:
|
||||
~Visitor() = default;
|
||||
};
|
||||
|
||||
template <typename Type>
|
||||
@ -76,6 +77,9 @@ public:
|
||||
using List = TypeList<Type>;
|
||||
|
||||
virtual void visit(Type &) = 0;
|
||||
|
||||
protected:
|
||||
~Visitor() = default;
|
||||
};
|
||||
|
||||
template <typename Type, typename ... Types>
|
||||
@ -86,6 +90,9 @@ public:
|
||||
using Visitor<Types ...>::visit;
|
||||
|
||||
virtual void visit(Type &) = 0;
|
||||
|
||||
protected:
|
||||
~Visitor() = default;
|
||||
};
|
||||
|
||||
|
||||
@ -95,6 +102,8 @@ class VisitorImplHelper;
|
||||
template <typename Derived, typename VisitorBase>
|
||||
class VisitorImplHelper<Derived, VisitorBase> : public VisitorBase
|
||||
{
|
||||
protected:
|
||||
~VisitorImplHelper() = default;
|
||||
};
|
||||
|
||||
template <typename Derived, typename VisitorBase, typename Type>
|
||||
@ -111,6 +120,8 @@ protected:
|
||||
throw Exception("visitImpl(" + demangle(typeid(T).name()) + " &)" + " is not implemented for class"
|
||||
+ demangle(typeid(Derived).name()), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
~VisitorImplHelper() = default;
|
||||
};
|
||||
|
||||
template <typename Derived, typename VisitorBase, typename Type, typename ... Types>
|
||||
@ -128,6 +139,8 @@ protected:
|
||||
throw Exception("visitImpl(" + demangle(typeid(T).name()) + " &)" + " is not implemented for class"
|
||||
+ demangle(typeid(Derived).name()), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
~VisitorImplHelper() = default;
|
||||
};
|
||||
|
||||
template <typename Derived, typename VisitorBase>
|
||||
@ -140,6 +153,8 @@ class VisitorImpl : public
|
||||
>::Type
|
||||
>::Type
|
||||
{
|
||||
protected:
|
||||
~VisitorImpl() = default;
|
||||
};
|
||||
|
||||
template <typename Derived, typename Base, typename Visitor>
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <vector>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Poco/Unicode.h>
|
||||
#include <Common/StringSearcher.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <vector>
|
||||
|
@ -1,7 +1,7 @@
|
||||
#include <Common/ZooKeeper/TestKeeper.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
|
@ -200,6 +200,18 @@ ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std
|
||||
init(args.implementation, args.hosts, args.identity, args.session_timeout_ms, args.operation_timeout_ms, args.chroot);
|
||||
}
|
||||
|
||||
bool ZooKeeper::configChanged(const Poco::Util::AbstractConfiguration & config, const std::string & config_name) const
|
||||
{
|
||||
ZooKeeperArgs args(config, config_name);
|
||||
|
||||
// skip reload testkeeper cause it's for test and data in memory
|
||||
if (args.implementation == implementation && implementation == "testkeeper")
|
||||
return false;
|
||||
|
||||
return std::tie(args.implementation, args.hosts, args.identity, args.session_timeout_ms, args.operation_timeout_ms, args.chroot)
|
||||
!= std::tie(implementation, hosts, identity, session_timeout_ms, operation_timeout_ms, chroot);
|
||||
}
|
||||
|
||||
|
||||
static Coordination::WatchCallback callbackForEvent(const EventPtr & watch)
|
||||
{
|
||||
|
@ -56,7 +56,7 @@ public:
|
||||
int32_t session_timeout_ms_ = DEFAULT_SESSION_TIMEOUT,
|
||||
int32_t operation_timeout_ms_ = DEFAULT_OPERATION_TIMEOUT,
|
||||
const std::string & chroot_ = "",
|
||||
const std::string & implementation = "zookeeper");
|
||||
const std::string & implementation_ = "zookeeper");
|
||||
|
||||
/** Config of the form:
|
||||
<zookeeper>
|
||||
@ -87,6 +87,8 @@ public:
|
||||
/// This object remains unchanged, and the new session is returned.
|
||||
Ptr startNewSession() const;
|
||||
|
||||
bool configChanged(const Poco::Util::AbstractConfiguration & config, const std::string & config_name) const;
|
||||
|
||||
/// Returns true, if the session has expired.
|
||||
bool expired();
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <filesystem>
|
||||
|
@ -3,7 +3,7 @@
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
|
||||
#include <common/types.h>
|
||||
#include <common/extended_types.h>
|
||||
|
||||
// Also defined in Core/Defines.h
|
||||
#if !defined(NO_SANITIZE_UNDEFINED)
|
||||
|
@ -1,7 +1,7 @@
|
||||
#include <Common/isLocalAddress.h>
|
||||
|
||||
#include <cstring>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/Net/NetworkInterface.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
|
@ -14,7 +14,7 @@
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
#pragma once
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <vector>
|
||||
namespace DB
|
||||
{
|
||||
|
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
/** Returns a number suitable as seed for PRNG. Use clock_gettime, pid and so on. */
|
||||
DB::UInt64 randomSeed();
|
||||
|
201
src/Common/remapExecutable.cpp
Normal file
201
src/Common/remapExecutable.cpp
Normal file
@ -0,0 +1,201 @@
|
||||
#if defined(__linux__) && defined(__amd64__) && defined(__SSE2__) && !defined(SANITIZER) && defined(NDEBUG) && !defined(SPLIT_SHARED_LIBRARIES)
|
||||
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
#include <emmintrin.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/hex.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
#include "remapExecutable.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int CANNOT_ALLOCATE_MEMORY;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
uintptr_t readAddressHex(DB::ReadBuffer & in)
|
||||
{
|
||||
uintptr_t res = 0;
|
||||
while (!in.eof())
|
||||
{
|
||||
if (isHexDigit(*in.position()))
|
||||
{
|
||||
res *= 16;
|
||||
res += unhex(*in.position());
|
||||
++in.position();
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
/** Find the address and size of the mapped memory region pointed by ptr.
|
||||
*/
|
||||
std::pair<void *, size_t> getMappedArea(void * ptr)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
uintptr_t uintptr = reinterpret_cast<uintptr_t>(ptr);
|
||||
ReadBufferFromFile in("/proc/self/maps");
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
uintptr_t begin = readAddressHex(in);
|
||||
assertChar('-', in);
|
||||
uintptr_t end = readAddressHex(in);
|
||||
skipToNextLineOrEOF(in);
|
||||
|
||||
if (begin <= uintptr && uintptr < end)
|
||||
return {reinterpret_cast<void *>(begin), end - begin};
|
||||
}
|
||||
|
||||
throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
||||
__attribute__((__noinline__)) int64_t our_syscall(...)
|
||||
{
|
||||
__asm__ __volatile__ (R"(
|
||||
movq %%rdi,%%rax;
|
||||
movq %%rsi,%%rdi;
|
||||
movq %%rdx,%%rsi;
|
||||
movq %%rcx,%%rdx;
|
||||
movq %%r8,%%r10;
|
||||
movq %%r9,%%r8;
|
||||
movq 8(%%rsp),%%r9;
|
||||
syscall;
|
||||
ret
|
||||
)" : : : "memory");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
__attribute__((__noinline__)) void remapToHugeStep3(void * scratch, size_t size, size_t offset)
|
||||
{
|
||||
/// The function should not use the stack, otherwise various optimizations, including "omit-frame-pointer" may break the code.
|
||||
|
||||
/// Unmap the scratch area.
|
||||
our_syscall(SYS_munmap, scratch, size);
|
||||
|
||||
/** The return address of this function is pointing to scratch area (because it was called from there).
|
||||
* But the scratch area no longer exists. We should correct the return address by subtracting the offset.
|
||||
*/
|
||||
__asm__ __volatile__("subq %0, 8(%%rsp)" : : "r"(offset) : "memory");
|
||||
}
|
||||
|
||||
|
||||
__attribute__((__noinline__)) void remapToHugeStep2(void * begin, size_t size, void * scratch)
|
||||
{
|
||||
/** Unmap old memory region with the code of our program.
|
||||
* Our instruction pointer is located inside scratch area and this function can execute after old code is unmapped.
|
||||
* But it cannot call any other functions because they are not available at usual addresses
|
||||
* - that's why we have to use "our_syscall" function and a substitution for memcpy.
|
||||
* (Relative addressing may continue to work but we should not assume that).
|
||||
*/
|
||||
|
||||
int64_t offset = reinterpret_cast<intptr_t>(scratch) - reinterpret_cast<intptr_t>(begin);
|
||||
int64_t (*syscall_func)(...) = reinterpret_cast<int64_t (*)(...)>(reinterpret_cast<intptr_t>(our_syscall) + offset);
|
||||
|
||||
int64_t munmap_res = syscall_func(SYS_munmap, begin, size);
|
||||
if (munmap_res != 0)
|
||||
return;
|
||||
|
||||
/// Map new anonymous memory region in place of old region with code.
|
||||
|
||||
int64_t mmap_res = syscall_func(SYS_mmap, begin, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
|
||||
if (-1 == mmap_res)
|
||||
syscall_func(SYS_exit, 1);
|
||||
|
||||
/// As the memory region is anonymous, we can do madvise with MADV_HUGEPAGE.
|
||||
|
||||
syscall_func(SYS_madvise, begin, size, MADV_HUGEPAGE);
|
||||
|
||||
/// Copy the code from scratch area to the old memory location.
|
||||
|
||||
{
|
||||
__m128i * __restrict dst = reinterpret_cast<__m128i *>(begin);
|
||||
const __m128i * __restrict src = reinterpret_cast<const __m128i *>(scratch);
|
||||
const __m128i * __restrict src_end = reinterpret_cast<const __m128i *>(reinterpret_cast<const char *>(scratch) + size);
|
||||
while (src < src_end)
|
||||
{
|
||||
_mm_storeu_si128(dst, _mm_loadu_si128(src));
|
||||
|
||||
++dst;
|
||||
++src;
|
||||
}
|
||||
}
|
||||
|
||||
/// Make the memory area with the code executable and non-writable.
|
||||
|
||||
syscall_func(SYS_mprotect, begin, size, PROT_READ | PROT_EXEC);
|
||||
|
||||
/** Step 3 function should unmap the scratch area.
|
||||
* The currently executed code is located in the scratch area and cannot be removed here.
|
||||
* We have to call another function and use its address from the original location (not in scratch area).
|
||||
* To do it, we obtain its pointer and call by pointer.
|
||||
*/
|
||||
|
||||
void(* volatile step3)(void*, size_t, size_t) = remapToHugeStep3;
|
||||
step3(scratch, size, offset);
|
||||
}
|
||||
|
||||
|
||||
__attribute__((__noinline__)) void remapToHugeStep1(void * begin, size_t size)
|
||||
{
|
||||
/// Allocate scratch area and copy the code there.
|
||||
|
||||
void * scratch = mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (MAP_FAILED == scratch)
|
||||
throwFromErrno(fmt::format("Cannot mmap {} bytes", size), ErrorCodes::CANNOT_ALLOCATE_MEMORY);
|
||||
|
||||
memcpy(scratch, begin, size);
|
||||
|
||||
/// Offset to the scratch area from previous location.
|
||||
|
||||
int64_t offset = reinterpret_cast<intptr_t>(scratch) - reinterpret_cast<intptr_t>(begin);
|
||||
|
||||
/// Jump to the next function inside the scratch area.
|
||||
|
||||
reinterpret_cast<void(*)(void*, size_t, void*)>(reinterpret_cast<intptr_t>(remapToHugeStep2) + offset)(begin, size, scratch);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void remapExecutable()
|
||||
{
|
||||
auto [begin, size] = getMappedArea(reinterpret_cast<void *>(remapExecutable));
|
||||
remapToHugeStep1(begin, size);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void remapExecutable() {}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
7
src/Common/remapExecutable.h
Normal file
7
src/Common/remapExecutable.h
Normal file
@ -0,0 +1,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// This function tries to reallocate the code of the running program in a more efficient way.
|
||||
void remapExecutable();
|
||||
|
||||
}
|
@ -3,7 +3,7 @@
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/HashTable/FixedHashMap.h>
|
||||
#include <Common/Arena.h>
|
||||
|
@ -1,5 +1,5 @@
|
||||
#include <iostream>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <IO/copyData.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
|
@ -12,7 +12,7 @@
|
||||
//#define DBMS_HASH_MAP_COUNT_COLLISIONS
|
||||
//#define DBMS_HASH_MAP_DEBUG_RESIZES
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <Compression/CompressedReadBuffer.h>
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
|
@ -1,5 +1,5 @@
|
||||
#include <Common/PODArray.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
#include <iostream>
|
||||
|
||||
#define ASSERT_CHECK(cond, res) \
|
||||
|
@ -74,6 +74,7 @@ SRCS(
|
||||
QueryProfiler.cpp
|
||||
quoteString.cpp
|
||||
randomSeed.cpp
|
||||
remapExecutable.cpp
|
||||
RemoteHostFilter.cpp
|
||||
renameat2.cpp
|
||||
RWLock.cpp
|
||||
|
@ -2,7 +2,7 @@
|
||||
#include <string.h>
|
||||
|
||||
#include <common/unaligned.h>
|
||||
#include <Core/Types.h>
|
||||
#include <common/types.h>
|
||||
|
||||
#include "CompressedWriteBuffer.h"
|
||||
#include <Compression/CompressionFactory.h>
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user