mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' into check_flaky_test
This commit is contained in:
alesapin
commit
265f84fd88
4
.gitmodules
vendored
4
.gitmodules
vendored
@ -186,3 +186,7 @@
|
||||
path = contrib/cyrus-sasl
|
||||
url = https://github.com/cyrusimap/cyrus-sasl
|
||||
branch = cyrus-sasl-2.1
|
||||
[submodule "contrib/croaring"]
|
||||
path = contrib/croaring
|
||||
url = https://github.com/RoaringBitmap/CRoaring
|
||||
branch = v0.2.66
|
||||
|
||||
339
base/glibc-compatibility/musl/lgammal.c
Normal file
339
base/glibc-compatibility/musl/lgammal.c
Normal file
@ -0,0 +1,339 @@
|
||||
/* origin: OpenBSD /usr/src/lib/libm/src/ld80/e_lgammal.c */
|
||||
/*
|
||||
* ====================================================
|
||||
* Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
|
||||
*
|
||||
* Developed at SunPro, a Sun Microsystems, Inc. business.
|
||||
* Permission to use, copy, modify, and distribute this
|
||||
* software is freely granted, provided that this notice
|
||||
* is preserved.
|
||||
* ====================================================
|
||||
*/
|
||||
/*
|
||||
* Copyright (c) 2008 Stephen L. Moshier <steve@moshier.net>
|
||||
*
|
||||
* Permission to use, copy, modify, and distribute this software for any
|
||||
* purpose with or without fee is hereby granted, provided that the above
|
||||
* copyright notice and this permission notice appear in all copies.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
/* lgammal(x)
|
||||
* Reentrant version of the logarithm of the Gamma function
|
||||
* with user provide pointer for the sign of Gamma(x).
|
||||
*
|
||||
* Method:
|
||||
* 1. Argument Reduction for 0 < x <= 8
|
||||
* Since gamma(1+s)=s*gamma(s), for x in [0,8], we may
|
||||
* reduce x to a number in [1.5,2.5] by
|
||||
* lgamma(1+s) = log(s) + lgamma(s)
|
||||
* for example,
|
||||
* lgamma(7.3) = log(6.3) + lgamma(6.3)
|
||||
* = log(6.3*5.3) + lgamma(5.3)
|
||||
* = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3)
|
||||
* 2. Polynomial approximation of lgamma around its
|
||||
* minimun ymin=1.461632144968362245 to maintain monotonicity.
|
||||
* On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use
|
||||
* Let z = x-ymin;
|
||||
* lgamma(x) = -1.214862905358496078218 + z^2*poly(z)
|
||||
* 2. Rational approximation in the primary interval [2,3]
|
||||
* We use the following approximation:
|
||||
* s = x-2.0;
|
||||
* lgamma(x) = 0.5*s + s*P(s)/Q(s)
|
||||
* Our algorithms are based on the following observation
|
||||
*
|
||||
* zeta(2)-1 2 zeta(3)-1 3
|
||||
* lgamma(2+s) = s*(1-Euler) + --------- * s - --------- * s + ...
|
||||
* 2 3
|
||||
*
|
||||
* where Euler = 0.5771... is the Euler constant, which is very
|
||||
* close to 0.5.
|
||||
*
|
||||
* 3. For x>=8, we have
|
||||
* lgamma(x)~(x-0.5)log(x)-x+0.5*log(2pi)+1/(12x)-1/(360x**3)+....
|
||||
* (better formula:
|
||||
* lgamma(x)~(x-0.5)*(log(x)-1)-.5*(log(2pi)-1) + ...)
|
||||
* Let z = 1/x, then we approximation
|
||||
* f(z) = lgamma(x) - (x-0.5)(log(x)-1)
|
||||
* by
|
||||
* 3 5 11
|
||||
* w = w0 + w1*z + w2*z + w3*z + ... + w6*z
|
||||
*
|
||||
* 4. For negative x, since (G is gamma function)
|
||||
* -x*G(-x)*G(x) = pi/sin(pi*x),
|
||||
* we have
|
||||
* G(x) = pi/(sin(pi*x)*(-x)*G(-x))
|
||||
* since G(-x) is positive, sign(G(x)) = sign(sin(pi*x)) for x<0
|
||||
* Hence, for x<0, signgam = sign(sin(pi*x)) and
|
||||
* lgamma(x) = log(|Gamma(x)|)
|
||||
* = log(pi/(|x*sin(pi*x)|)) - lgamma(-x);
|
||||
* Note: one should avoid compute pi*(-x) directly in the
|
||||
* computation of sin(pi*(-x)).
|
||||
*
|
||||
* 5. Special Cases
|
||||
* lgamma(2+s) ~ s*(1-Euler) for tiny s
|
||||
* lgamma(1)=lgamma(2)=0
|
||||
* lgamma(x) ~ -log(x) for tiny x
|
||||
* lgamma(0) = lgamma(inf) = inf
|
||||
* lgamma(-integer) = +-inf
|
||||
*
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <math.h>
|
||||
#include "libm.h"
|
||||
|
||||
|
||||
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
|
||||
double lgamma_r(double x, int *sg);
|
||||
|
||||
long double lgammal_r(long double x, int *sg)
|
||||
{
|
||||
return lgamma_r(x, sg);
|
||||
}
|
||||
#elif LDBL_MANT_DIG == 64 && LDBL_MAX_EXP == 16384
|
||||
|
||||
static const long double pi = 3.14159265358979323846264L,
|
||||
|
||||
/* lgam(1+x) = 0.5 x + x a(x)/b(x)
|
||||
-0.268402099609375 <= x <= 0
|
||||
peak relative error 6.6e-22 */
|
||||
a0 = -6.343246574721079391729402781192128239938E2L,
|
||||
a1 = 1.856560238672465796768677717168371401378E3L,
|
||||
a2 = 2.404733102163746263689288466865843408429E3L,
|
||||
a3 = 8.804188795790383497379532868917517596322E2L,
|
||||
a4 = 1.135361354097447729740103745999661157426E2L,
|
||||
a5 = 3.766956539107615557608581581190400021285E0L,
|
||||
|
||||
b0 = 8.214973713960928795704317259806842490498E3L,
|
||||
b1 = 1.026343508841367384879065363925870888012E4L,
|
||||
b2 = 4.553337477045763320522762343132210919277E3L,
|
||||
b3 = 8.506975785032585797446253359230031874803E2L,
|
||||
b4 = 6.042447899703295436820744186992189445813E1L,
|
||||
/* b5 = 1.000000000000000000000000000000000000000E0 */
|
||||
|
||||
|
||||
tc = 1.4616321449683623412626595423257213284682E0L,
|
||||
tf = -1.2148629053584961146050602565082954242826E-1, /* double precision */
|
||||
/* tt = (tail of tf), i.e. tf + tt has extended precision. */
|
||||
tt = 3.3649914684731379602768989080467587736363E-18L,
|
||||
/* lgam ( 1.4616321449683623412626595423257213284682E0 ) =
|
||||
-1.2148629053584960809551455717769158215135617312999903886372437313313530E-1 */
|
||||
|
||||
/* lgam (x + tc) = tf + tt + x g(x)/h(x)
|
||||
-0.230003726999612341262659542325721328468 <= x
|
||||
<= 0.2699962730003876587373404576742786715318
|
||||
peak relative error 2.1e-21 */
|
||||
g0 = 3.645529916721223331888305293534095553827E-18L,
|
||||
g1 = 5.126654642791082497002594216163574795690E3L,
|
||||
g2 = 8.828603575854624811911631336122070070327E3L,
|
||||
g3 = 5.464186426932117031234820886525701595203E3L,
|
||||
g4 = 1.455427403530884193180776558102868592293E3L,
|
||||
g5 = 1.541735456969245924860307497029155838446E2L,
|
||||
g6 = 4.335498275274822298341872707453445815118E0L,
|
||||
|
||||
h0 = 1.059584930106085509696730443974495979641E4L,
|
||||
h1 = 2.147921653490043010629481226937850618860E4L,
|
||||
h2 = 1.643014770044524804175197151958100656728E4L,
|
||||
h3 = 5.869021995186925517228323497501767586078E3L,
|
||||
h4 = 9.764244777714344488787381271643502742293E2L,
|
||||
h5 = 6.442485441570592541741092969581997002349E1L,
|
||||
/* h6 = 1.000000000000000000000000000000000000000E0 */
|
||||
|
||||
|
||||
/* lgam (x+1) = -0.5 x + x u(x)/v(x)
|
||||
-0.100006103515625 <= x <= 0.231639862060546875
|
||||
peak relative error 1.3e-21 */
|
||||
u0 = -8.886217500092090678492242071879342025627E1L,
|
||||
u1 = 6.840109978129177639438792958320783599310E2L,
|
||||
u2 = 2.042626104514127267855588786511809932433E3L,
|
||||
u3 = 1.911723903442667422201651063009856064275E3L,
|
||||
u4 = 7.447065275665887457628865263491667767695E2L,
|
||||
u5 = 1.132256494121790736268471016493103952637E2L,
|
||||
u6 = 4.484398885516614191003094714505960972894E0L,
|
||||
|
||||
v0 = 1.150830924194461522996462401210374632929E3L,
|
||||
v1 = 3.399692260848747447377972081399737098610E3L,
|
||||
v2 = 3.786631705644460255229513563657226008015E3L,
|
||||
v3 = 1.966450123004478374557778781564114347876E3L,
|
||||
v4 = 4.741359068914069299837355438370682773122E2L,
|
||||
v5 = 4.508989649747184050907206782117647852364E1L,
|
||||
/* v6 = 1.000000000000000000000000000000000000000E0 */
|
||||
|
||||
|
||||
/* lgam (x+2) = .5 x + x s(x)/r(x)
|
||||
0 <= x <= 1
|
||||
peak relative error 7.2e-22 */
|
||||
s0 = 1.454726263410661942989109455292824853344E6L,
|
||||
s1 = -3.901428390086348447890408306153378922752E6L,
|
||||
s2 = -6.573568698209374121847873064292963089438E6L,
|
||||
s3 = -3.319055881485044417245964508099095984643E6L,
|
||||
s4 = -7.094891568758439227560184618114707107977E5L,
|
||||
s5 = -6.263426646464505837422314539808112478303E4L,
|
||||
s6 = -1.684926520999477529949915657519454051529E3L,
|
||||
|
||||
r0 = -1.883978160734303518163008696712983134698E7L,
|
||||
r1 = -2.815206082812062064902202753264922306830E7L,
|
||||
r2 = -1.600245495251915899081846093343626358398E7L,
|
||||
r3 = -4.310526301881305003489257052083370058799E6L,
|
||||
r4 = -5.563807682263923279438235987186184968542E5L,
|
||||
r5 = -3.027734654434169996032905158145259713083E4L,
|
||||
r6 = -4.501995652861105629217250715790764371267E2L,
|
||||
/* r6 = 1.000000000000000000000000000000000000000E0 */
|
||||
|
||||
|
||||
/* lgam(x) = ( x - 0.5 ) * log(x) - x + LS2PI + 1/x w(1/x^2)
|
||||
x >= 8
|
||||
Peak relative error 1.51e-21
|
||||
w0 = LS2PI - 0.5 */
|
||||
w0 = 4.189385332046727417803e-1L,
|
||||
w1 = 8.333333333333331447505E-2L,
|
||||
w2 = -2.777777777750349603440E-3L,
|
||||
w3 = 7.936507795855070755671E-4L,
|
||||
w4 = -5.952345851765688514613E-4L,
|
||||
w5 = 8.412723297322498080632E-4L,
|
||||
w6 = -1.880801938119376907179E-3L,
|
||||
w7 = 4.885026142432270781165E-3L;
|
||||
|
||||
|
||||
long double lgammal_r(long double x, int *sg) {
|
||||
long double t, y, z, nadj, p, p1, p2, q, r, w;
|
||||
union ldshape u = {x};
|
||||
uint32_t ix = (u.i.se & 0x7fffU)<<16 | u.i.m>>48;
|
||||
int sign = u.i.se >> 15;
|
||||
int i;
|
||||
|
||||
*sg = 1;
|
||||
|
||||
/* purge off +-inf, NaN, +-0, tiny and negative arguments */
|
||||
if (ix >= 0x7fff0000)
|
||||
return x * x;
|
||||
if (ix < 0x3fc08000) { /* |x|<2**-63, return -log(|x|) */
|
||||
if (sign) {
|
||||
*sg = -1;
|
||||
x = -x;
|
||||
}
|
||||
return -logl(x);
|
||||
}
|
||||
if (sign) {
|
||||
x = -x;
|
||||
t = sin(pi * x);
|
||||
if (t == 0.0)
|
||||
return 1.0 / (x-x); /* -integer */
|
||||
if (t > 0.0)
|
||||
*sg = -1;
|
||||
else
|
||||
t = -t;
|
||||
nadj = logl(pi / (t * x));
|
||||
}
|
||||
|
||||
/* purge off 1 and 2 (so the sign is ok with downward rounding) */
|
||||
if ((ix == 0x3fff8000 || ix == 0x40008000) && u.i.m == 0) {
|
||||
r = 0;
|
||||
} else if (ix < 0x40008000) { /* x < 2.0 */
|
||||
if (ix <= 0x3ffee666) { /* 8.99993896484375e-1 */
|
||||
/* lgamma(x) = lgamma(x+1) - log(x) */
|
||||
r = -logl(x);
|
||||
if (ix >= 0x3ffebb4a) { /* 7.31597900390625e-1 */
|
||||
y = x - 1.0;
|
||||
i = 0;
|
||||
} else if (ix >= 0x3ffced33) { /* 2.31639862060546875e-1 */
|
||||
y = x - (tc - 1.0);
|
||||
i = 1;
|
||||
} else { /* x < 0.23 */
|
||||
y = x;
|
||||
i = 2;
|
||||
}
|
||||
} else {
|
||||
r = 0.0;
|
||||
if (ix >= 0x3fffdda6) { /* 1.73162841796875 */
|
||||
/* [1.7316,2] */
|
||||
y = x - 2.0;
|
||||
i = 0;
|
||||
} else if (ix >= 0x3fff9da6) { /* 1.23162841796875 */
|
||||
/* [1.23,1.73] */
|
||||
y = x - tc;
|
||||
i = 1;
|
||||
} else {
|
||||
/* [0.9, 1.23] */
|
||||
y = x - 1.0;
|
||||
i = 2;
|
||||
}
|
||||
}
|
||||
switch (i) {
|
||||
case 0:
|
||||
p1 = a0 + y * (a1 + y * (a2 + y * (a3 + y * (a4 + y * a5))));
|
||||
p2 = b0 + y * (b1 + y * (b2 + y * (b3 + y * (b4 + y))));
|
||||
r += 0.5 * y + y * p1/p2;
|
||||
break;
|
||||
case 1:
|
||||
p1 = g0 + y * (g1 + y * (g2 + y * (g3 + y * (g4 + y * (g5 + y * g6)))));
|
||||
p2 = h0 + y * (h1 + y * (h2 + y * (h3 + y * (h4 + y * (h5 + y)))));
|
||||
p = tt + y * p1/p2;
|
||||
r += (tf + p);
|
||||
break;
|
||||
case 2:
|
||||
p1 = y * (u0 + y * (u1 + y * (u2 + y * (u3 + y * (u4 + y * (u5 + y * u6))))));
|
||||
p2 = v0 + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * (v5 + y)))));
|
||||
r += (-0.5 * y + p1 / p2);
|
||||
}
|
||||
} else if (ix < 0x40028000) { /* 8.0 */
|
||||
/* x < 8.0 */
|
||||
i = (int)x;
|
||||
y = x - (double)i;
|
||||
p = y * (s0 + y * (s1 + y * (s2 + y * (s3 + y * (s4 + y * (s5 + y * s6))))));
|
||||
q = r0 + y * (r1 + y * (r2 + y * (r3 + y * (r4 + y * (r5 + y * (r6 + y))))));
|
||||
r = 0.5 * y + p / q;
|
||||
z = 1.0;
|
||||
/* lgamma(1+s) = log(s) + lgamma(s) */
|
||||
switch (i) {
|
||||
case 7:
|
||||
z *= (y + 6.0); /* FALLTHRU */
|
||||
case 6:
|
||||
z *= (y + 5.0); /* FALLTHRU */
|
||||
case 5:
|
||||
z *= (y + 4.0); /* FALLTHRU */
|
||||
case 4:
|
||||
z *= (y + 3.0); /* FALLTHRU */
|
||||
case 3:
|
||||
z *= (y + 2.0); /* FALLTHRU */
|
||||
r += logl(z);
|
||||
break;
|
||||
}
|
||||
} else if (ix < 0x40418000) { /* 2^66 */
|
||||
/* 8.0 <= x < 2**66 */
|
||||
t = logl(x);
|
||||
z = 1.0 / x;
|
||||
y = z * z;
|
||||
w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * (w6 + y * w7))))));
|
||||
r = (x - 0.5) * (t - 1.0) + w;
|
||||
} else /* 2**66 <= x <= inf */
|
||||
r = x * (logl(x) - 1.0);
|
||||
if (sign)
|
||||
r = nadj - r;
|
||||
return r;
|
||||
}
|
||||
#elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384
|
||||
// TODO: broken implementation to make things compile
|
||||
double lgamma_r(double x, int *sg);
|
||||
|
||||
long double lgammal_r(long double x, int *sg)
|
||||
{
|
||||
return lgamma_r(x, sg);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
int signgam_lgammal;
|
||||
|
||||
long double lgammal(long double x)
|
||||
{
|
||||
return lgammal_r(x, &signgam_lgammal);
|
||||
}
|
||||
|
||||
2
contrib/CMakeLists.txt
vendored
2
contrib/CMakeLists.txt
vendored
@ -20,7 +20,6 @@ add_subdirectory (boost-cmake)
|
||||
add_subdirectory (cctz-cmake)
|
||||
add_subdirectory (consistent-hashing-sumbur)
|
||||
add_subdirectory (consistent-hashing)
|
||||
add_subdirectory (croaring)
|
||||
add_subdirectory (FastMemcpy)
|
||||
add_subdirectory (hyperscan-cmake)
|
||||
add_subdirectory (jemalloc-cmake)
|
||||
@ -34,6 +33,7 @@ add_subdirectory (ryu-cmake)
|
||||
add_subdirectory (unixodbc-cmake)
|
||||
|
||||
add_subdirectory (poco-cmake)
|
||||
add_subdirectory (croaring-cmake)
|
||||
|
||||
|
||||
# TODO: refactor the contrib libraries below this comment.
|
||||
|
||||
1
contrib/croaring
vendored
Submodule
1
contrib/croaring
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 5f20740ec0de5e153e8f4cb2ab91814e8b291a14
|
||||
25
contrib/croaring-cmake/CMakeLists.txt
Normal file
25
contrib/croaring-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,25 @@
|
||||
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/croaring)
|
||||
|
||||
set(SRCS
|
||||
${LIBRARY_DIR}/src/array_util.c
|
||||
${LIBRARY_DIR}/src/bitset_util.c
|
||||
${LIBRARY_DIR}/src/containers/array.c
|
||||
${LIBRARY_DIR}/src/containers/bitset.c
|
||||
${LIBRARY_DIR}/src/containers/containers.c
|
||||
${LIBRARY_DIR}/src/containers/convert.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_intersection.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_union.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_equal.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_subset.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_negation.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_xor.c
|
||||
${LIBRARY_DIR}/src/containers/mixed_andnot.c
|
||||
${LIBRARY_DIR}/src/containers/run.c
|
||||
${LIBRARY_DIR}/src/roaring.c
|
||||
${LIBRARY_DIR}/src/roaring_priority_queue.c
|
||||
${LIBRARY_DIR}/src/roaring_array.c)
|
||||
|
||||
add_library(roaring ${SRCS})
|
||||
|
||||
target_include_directories(roaring PRIVATE ${LIBRARY_DIR}/include/roaring)
|
||||
target_include_directories(roaring SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include)
|
||||
@ -1,6 +0,0 @@
|
||||
add_library(roaring
|
||||
roaring.c
|
||||
roaring/roaring.h
|
||||
roaring/roaring.hh)
|
||||
|
||||
target_include_directories (roaring SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
@ -1,202 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016 The CRoaring authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
download from https://github.com/RoaringBitmap/CRoaring/archive/v0.2.57.tar.gz
|
||||
and use ./amalgamation.sh generate
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -53,6 +53,7 @@ RUN apt-get update \
|
||||
ninja-build \
|
||||
psmisc \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-termcolor \
|
||||
@ -62,6 +63,8 @@ RUN apt-get update \
|
||||
unixodbc \
|
||||
--yes --no-install-recommends
|
||||
|
||||
RUN pip3 install numpy scipy pandas
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@ FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}")
|
||||
FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}")
|
||||
FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}")
|
||||
FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}")
|
||||
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
|
||||
|
||||
# Export these variables, so that all subsequent invocations of the script
|
||||
# use them, and not try to guess them anew, which leads to weird effects.
|
||||
@ -28,6 +29,7 @@ export FASTTEST_SOURCE
|
||||
export FASTTEST_BUILD
|
||||
export FASTTEST_DATA
|
||||
export FASTTEST_OUT
|
||||
export PATH
|
||||
|
||||
server_pid=none
|
||||
|
||||
@ -125,7 +127,7 @@ function clone_submodules
|
||||
(
|
||||
cd "$FASTTEST_SOURCE"
|
||||
|
||||
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11)
|
||||
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring)
|
||||
|
||||
git submodule sync
|
||||
git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}"
|
||||
@ -137,7 +139,14 @@ git submodule foreach git clean -xfd
|
||||
|
||||
function run_cmake
|
||||
{
|
||||
CMAKE_LIBS_CONFIG=("-DENABLE_LIBRARIES=0" "-DENABLE_TESTS=0" "-DENABLE_UTILS=0" "-DENABLE_EMBEDDED_COMPILER=0" "-DENABLE_THINLTO=0" "-DUSE_UNWIND=1")
|
||||
CMAKE_LIBS_CONFIG=(
|
||||
"-DENABLE_LIBRARIES=0"
|
||||
"-DENABLE_TESTS=0"
|
||||
"-DENABLE_UTILS=0"
|
||||
"-DENABLE_EMBEDDED_COMPILER=0"
|
||||
"-DENABLE_THINLTO=0"
|
||||
"-DUSE_UNWIND=1"
|
||||
)
|
||||
|
||||
# TODO remove this? we don't use ccache anyway. An option would be to download it
|
||||
# from S3 simultaneously with cloning.
|
||||
@ -259,7 +268,12 @@ TESTS_TO_SKIP=(
|
||||
00974_query_profiler
|
||||
|
||||
# Look at DistributedFilesToInsert, so cannot run in parallel.
|
||||
01460_DistributedFilesToInsert
|
||||
01457_DistributedFilesToInsert
|
||||
|
||||
01541_max_memory_usage_for_user
|
||||
|
||||
# Require python libraries like scipy, pandas and numpy
|
||||
01322_ttest_scipy
|
||||
)
|
||||
|
||||
time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
|
||||
@ -329,8 +343,6 @@ case "$stage" in
|
||||
;&
|
||||
"build")
|
||||
build
|
||||
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
|
||||
export PATH
|
||||
;&
|
||||
"configure")
|
||||
# The `install_log.txt` is also needed for compatibility with old CI task --
|
||||
|
||||
@ -9,6 +9,7 @@ RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
bash \
|
||||
curl \
|
||||
dmidecode \
|
||||
g++ \
|
||||
gdb \
|
||||
git \
|
||||
@ -37,7 +38,18 @@ RUN apt-get update \
|
||||
|
||||
COPY * /
|
||||
|
||||
CMD /entrypoint.sh
|
||||
# Bind everything to one NUMA node, if there's more than one. Theoretically the
|
||||
# node #0 should be less stable because of system interruptions. We bind
|
||||
# randomly to node 1 or 0 to gather some statistics on that. We have to bind
|
||||
# both servers and the tmpfs on which the database is stored. How to do it
|
||||
# through Yandex Sandbox API is unclear, but by default tmpfs uses
|
||||
# 'process allocation policy', not sure which process but hopefully the one that
|
||||
# writes to it, so just bind the downloader script as well. We could also try to
|
||||
# remount it with proper options in Sandbox task.
|
||||
# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
|
||||
# Double-escaped backslashes are a tribute to the engineering wonder of docker --
|
||||
# it gives '/bin/sh: 1: [bash,: not found' otherwise.
|
||||
CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"]
|
||||
|
||||
# docker run --network=host --volume <workspace>:/workspace --volume=<output>:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison
|
||||
|
||||
|
||||
@ -77,20 +77,33 @@ function restart
|
||||
while killall clickhouse-server; do echo . ; sleep 1 ; done
|
||||
echo all killed
|
||||
|
||||
# Disable percpu arenas because they segfault when the process is bound to
|
||||
# a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939
|
||||
#
|
||||
# About the jemalloc settings:
|
||||
# https://github.com/jemalloc/jemalloc/wiki/Getting-Started
|
||||
export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true"
|
||||
|
||||
set -m # Spawn servers in their own process groups
|
||||
|
||||
left/clickhouse-server --config-file=left/config/config.xml -- --path left/db --user_files_path left/db/user_files &>> left-server-log.log &
|
||||
left/clickhouse-server --config-file=left/config/config.xml \
|
||||
-- --path left/db --user_files_path left/db/user_files \
|
||||
&>> left-server-log.log &
|
||||
left_pid=$!
|
||||
kill -0 $left_pid
|
||||
disown $left_pid
|
||||
|
||||
right/clickhouse-server --config-file=right/config/config.xml -- --path right/db --user_files_path right/db/user_files &>> right-server-log.log &
|
||||
right/clickhouse-server --config-file=right/config/config.xml \
|
||||
-- --path right/db --user_files_path right/db/user_files \
|
||||
&>> right-server-log.log &
|
||||
right_pid=$!
|
||||
kill -0 $right_pid
|
||||
disown $right_pid
|
||||
|
||||
set +m
|
||||
|
||||
unset MALLOC_CONF
|
||||
|
||||
wait_for_server 9001 $left_pid
|
||||
echo left ok
|
||||
|
||||
@ -449,7 +462,12 @@ wait
|
||||
unset IFS
|
||||
)
|
||||
|
||||
parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
|
||||
# The comparison script might be bound to one NUMA node for better test
|
||||
# stability, and the calculation runs out of memory because of this. Use
|
||||
# all nodes.
|
||||
numactl --show
|
||||
numactl --cpunodebind=all --membind=all numactl --show
|
||||
numactl --cpunodebind=all --membind=all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
|
||||
|
||||
clickhouse-local --query "
|
||||
-- Join the metric names back to the metric statistics we've calculated, and make
|
||||
@ -1070,8 +1088,10 @@ case "$stage" in
|
||||
time configure
|
||||
;&
|
||||
"restart")
|
||||
numactl --show ||:
|
||||
numactl --hardware ||:
|
||||
lscpu ||:
|
||||
dmidecode -t 4 ||:
|
||||
time restart
|
||||
;&
|
||||
"run_tests")
|
||||
|
||||
@ -14,6 +14,9 @@
|
||||
we might also add time check to perf.py script.
|
||||
-->
|
||||
<max_execution_time>300</max_execution_time>
|
||||
|
||||
<!-- One NUMA node w/o hyperthreading -->
|
||||
<max_threads>20</max_threads>
|
||||
</default>
|
||||
</profiles>
|
||||
</yandex>
|
||||
|
||||
@ -16,6 +16,7 @@ RUN apt-get update -y \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-termcolor \
|
||||
python3-pip \
|
||||
qemu-user-static \
|
||||
sudo \
|
||||
telnet \
|
||||
@ -23,6 +24,8 @@ RUN apt-get update -y \
|
||||
unixodbc \
|
||||
wget
|
||||
|
||||
RUN pip3 install numpy scipy pandas
|
||||
|
||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
|
||||
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
|
||||
|
||||
@ -58,6 +58,7 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
python3-lxml \
|
||||
python3-requests \
|
||||
python3-termcolor \
|
||||
python3-pip \
|
||||
qemu-user-static \
|
||||
sudo \
|
||||
telnet \
|
||||
@ -68,6 +69,8 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
wget \
|
||||
zlib1g-dev
|
||||
|
||||
RUN pip3 install numpy scipy pandas
|
||||
|
||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
|
||||
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
|
||||
|
||||
@ -35,7 +35,7 @@ RUN apt-get update \
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN pip3 install urllib3 testflows==1.6.48 docker-compose docker dicttoxml kazoo tzlocal
|
||||
RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 17.09.1-ce
|
||||
@ -72,5 +72,5 @@ RUN set -x \
|
||||
VOLUME /var/lib/docker
|
||||
EXPOSE 2375
|
||||
ENTRYPOINT ["dockerd-entrypoint.sh"]
|
||||
CMD ["sh", "-c", "python3 regression.py --no-color --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"]
|
||||
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"]
|
||||
|
||||
|
||||
@ -195,7 +195,7 @@ Templates:
|
||||
|
||||
- [Function](_description_templates/template-function.md)
|
||||
- [Setting](_description_templates/template-setting.md)
|
||||
- [Table engine](_description_templates/template-table-engine.md)
|
||||
- [Database or Table engine](_description_templates/template-engine.md)
|
||||
- [System table](_description_templates/template-system-table.md)
|
||||
|
||||
|
||||
|
||||
@ -1,8 +1,14 @@
|
||||
# EngineName {#enginename}
|
||||
|
||||
- What the engine does.
|
||||
- What the Database/Table engine does.
|
||||
- Relations with other engines if they exist.
|
||||
|
||||
## Creating a Database {#creating-a-database}
|
||||
``` sql
|
||||
CREATE DATABASE ...
|
||||
```
|
||||
or
|
||||
|
||||
## Creating a Table {#creating-a-table}
|
||||
``` sql
|
||||
CREATE TABLE ...
|
||||
@ -10,12 +16,19 @@
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
**Query Clauses**
|
||||
**Query Clauses** (for Table engines only)
|
||||
|
||||
## Virtual columns {#virtual-columns}
|
||||
## Virtual columns {#virtual-columns} (for Table engines only)
|
||||
|
||||
List and virtual columns with description, if they exist.
|
||||
|
||||
## Data Types Support {#data_types-support} (for Database engines only)
|
||||
|
||||
| EngineName | ClickHouse |
|
||||
|-----------------------|------------------------------------|
|
||||
| NativeDataTypeName | [ClickHouseDataTypeName](link#) |
|
||||
|
||||
|
||||
## Specifics and recommendations {#specifics-and-recommendations}
|
||||
|
||||
Algorithms
|
||||
@ -47,6 +47,8 @@ select x; -- { serverError 49 }
|
||||
```
|
||||
This test ensures that the server returns an error with code 49 about unknown column `x`. If there is no error, or the error is different, the test will fail. If you want to ensure that an error occurs on the client side, use `clientError` annotation instead.
|
||||
|
||||
Do not check for a particular wording of error message, it may change in the future, and the test will needlessly break. Check only the error code. If the existing error code is not precise enough for your needs, consider adding a new one.
|
||||
|
||||
### Testing a Distributed Query
|
||||
|
||||
If you want to use distributed queries in functional tests, you can leverage `remote` table function with `127.0.0.{1..2}` addresses for the server to query itself; or you can use predefined test clusters in server configuration file like `test_shard_localhost`. Remember to add the words `shard` or `distributed` to the test name, so that it is ran in CI in correct configurations, where the server is configured to support distributed queries.
|
||||
|
||||
@ -77,6 +77,7 @@ toc_title: Adopters
|
||||
| <a href="https://rambler.ru" class="favicon">Rambler</a> | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) |
|
||||
| <a href="https://retell.cc/" class="favicon">Retell</a> | Speech synthesis | Analytics | — | — | [Blog Article, August 2020](https://vc.ru/services/153732-kak-sozdat-audiostati-na-vashem-sayte-i-zachem-eto-nuzhno) |
|
||||
| <a href="https://rspamd.com/" class="favicon">Rspamd</a> | Antispam | Analytics | — | — | [Official Website](https://rspamd.com/doc/modules/clickhouse.html) |
|
||||
| <a href="https://rusiem.com/en" class="favicon">RuSIEM</a> | SIEM | Main Product | — | — | [Official Website](https://rusiem.com/en/products/architecture) |
|
||||
| <a href="https://www.s7.ru" class="favicon">S7 Airlines</a> | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) |
|
||||
| <a href="https://www.scireum.de/" class="favicon">scireum GmbH</a> | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) |
|
||||
| <a href="https://segment.com/" class="favicon">Segment</a> | Data processing | Main product | 9 * i3en.3xlarge nodes 7.5TB NVME SSDs, 96GB Memory, 12 vCPUs | — | [Slides, 2019](https://slides.com/abraithwaite/segment-clickhouse) |
|
||||
|
||||
48
docs/en/operations/system-tables/crash-log.md
Normal file
48
docs/en/operations/system-tables/crash-log.md
Normal file
@ -0,0 +1,48 @@
|
||||
# system.crash_log {#system-tables_crash_log}
|
||||
|
||||
Contains information about stack traces for fatal errors. The table does not exist in the database by default, it is created only when fatal errors occur.
|
||||
|
||||
Columns:
|
||||
|
||||
- `event_date` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date of the event.
|
||||
- `event_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Time of the event.
|
||||
- `timestamp_ns` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Timestamp of the event with nanoseconds.
|
||||
- `signal` ([Int32](../../sql-reference/data-types/int-uint.md)) — Signal number.
|
||||
- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Thread ID.
|
||||
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Query ID.
|
||||
- `trace` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Stack trace at the moment of crash. Each element is a virtual memory address inside ClickHouse server process.
|
||||
- `trace_full` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Stack trace at the moment of crash. Each element contains a called method inside ClickHouse server process.
|
||||
- `version` ([String](../../sql-reference/data-types/string.md)) — ClickHouse server version.
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse server revision.
|
||||
- `build_id` ([String](../../sql-reference/data-types/string.md)) — BuildID that is generated by compiler.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.crash_log ORDER BY event_time DESC LIMIT 1;
|
||||
```
|
||||
|
||||
Result (not full):
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
event_date: 2020-10-14
|
||||
event_time: 2020-10-14 15:47:40
|
||||
timestamp_ns: 1602679660271312710
|
||||
signal: 11
|
||||
thread_id: 23624
|
||||
query_id: 428aab7c-8f5c-44e9-9607-d16b44467e69
|
||||
trace: [188531193,...]
|
||||
trace_full: ['3. DB::(anonymous namespace)::FunctionFormatReadableTimeDelta::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >&, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > const&, unsigned long, unsigned long) const @ 0xb3cc1f9 in /home/username/work/ClickHouse/build/programs/clickhouse',...]
|
||||
version: ClickHouse 20.11.1.1
|
||||
revision: 54442
|
||||
build_id:
|
||||
```
|
||||
|
||||
**See also**
|
||||
- [trace_log](../../operations/system-tables/trace_log.md) system table
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system-tables/crash-log)
|
||||
@ -15,12 +15,83 @@ Returns a single `String`-type ‘statement’ column, which contains a single v
|
||||
|
||||
## SHOW DATABASES {#show-databases}
|
||||
|
||||
Prints a list of all databases.
|
||||
|
||||
```sql
|
||||
SHOW DATABASES [INTO OUTFILE filename] [FORMAT format]
|
||||
SHOW DATABASES [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
Prints a list of all databases.
|
||||
This query is identical to `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`.
|
||||
This statement is identical to the query:
|
||||
|
||||
```sql
|
||||
SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
### Examples {#examples}
|
||||
|
||||
Getting database names, containing the symbols sequence 'de' in their names:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES LIKE '%de%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name────┐
|
||||
│ default │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Getting database names, containing symbols sequence 'de' in their names, in the case insensitive manner:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES ILIKE '%DE%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name────┐
|
||||
│ default │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Getting database names, not containing the symbols sequence 'de' in their names:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES NOT LIKE '%de%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ _temporary_and_external_tables │
|
||||
│ system │
|
||||
│ test │
|
||||
│ tutorial │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
Getting the first two rows from database names:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES LIMIT 2
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ _temporary_and_external_tables │
|
||||
│ default │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
### See Also {#see-also}
|
||||
|
||||
- [CREATE DATABASE](https://clickhouse.tech/docs/en/sql-reference/statements/create/database/#query-language-create-database)
|
||||
|
||||
## SHOW PROCESSLIST {#show-processlist}
|
||||
|
||||
@ -43,32 +114,85 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
|
||||
Displays a list of tables.
|
||||
|
||||
```sql
|
||||
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE '<pattern>' | WHERE expr] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
```
|
||||
|
||||
If the `FROM` clause is not specified, the query returns the list of tables from the current database.
|
||||
|
||||
You can get the same results as the `SHOW TABLES` query in the following way:
|
||||
This statement is identical to the query:
|
||||
|
||||
```sql
|
||||
SELECT name FROM system.tables WHERE database = <db> [AND name LIKE <pattern>] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
```
|
||||
|
||||
**Example**
|
||||
### Examples {#examples}
|
||||
|
||||
The following query selects the first two rows from the list of tables in the `system` database, whose names contain `co`.
|
||||
Getting table names, containing the symbols sequence 'user' in their names:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system LIKE '%co%' LIMIT 2
|
||||
SHOW TABLES FROM system LIKE '%user%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name─────────────┐
|
||||
│ user_directories │
|
||||
│ users │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
Getting table names, containing sequence 'user' in their names, in the case insensitive manner:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system ILIKE '%USER%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name─────────────┐
|
||||
│ user_directories │
|
||||
│ users │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
Getting table names, not containing the symbol sequence 's' in their names:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system NOT LIKE '%s%'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name─────────┐
|
||||
│ metric_log │
|
||||
│ metric_log_0 │
|
||||
│ metric_log_1 │
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
Getting the first two rows from table names:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system LIMIT 2
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ aggregate_function_combinators │
|
||||
│ collations │
|
||||
│ asynchronous_metric_log │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
### See Also {#see-also}
|
||||
|
||||
- [Create Tables](https://clickhouse.tech/docs/en/getting-started/tutorial/#create-tables)
|
||||
- [SHOW CREATE TABLE](https://clickhouse.tech/docs/en/sql-reference/statements/show/#show-create-table)
|
||||
|
||||
## SHOW DICTIONARIES {#show-dictionaries}
|
||||
|
||||
Displays a list of [external dictionaries](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
|
||||
|
||||
@ -87,7 +87,7 @@ In string literals, you need to escape at least `'` and `\`. Single quotes can b
|
||||
|
||||
### Compound {#compound}
|
||||
|
||||
Arrays are constructed with square brackets `[1, 2, 3]`. Nuples are constructed with round brackets `(1, 'Hello, world!', 2)`.
|
||||
Arrays are constructed with square brackets `[1, 2, 3]`. Tuples are constructed with round brackets `(1, 'Hello, world!', 2)`.
|
||||
Technically these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively.
|
||||
An array must consist of at least one item, and a tuple must have at least two items.
|
||||
There’s a separate case when tuples appear in the `IN` clause of a `SELECT` query. Query results can include tuples, but tuples can’t be saved to a database (except of tables with [Memory](../engines/table-engines/special/memory.md) engine).
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
|
||||
Репликация не зависит от шардирования. На каждом шарде репликация работает независимо.
|
||||
|
||||
Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../../engines/table-engines/mergetree-family/replication.md#query_language_queries_alter)).
|
||||
Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../../sql-reference/statements/alter/index.md#query_language_queries_alter)).
|
||||
|
||||
Запросы `CREATE`, `DROP`, `ATTACH`, `DETACH` и `RENAME` выполняются на одном сервере и не реплицируются:
|
||||
|
||||
|
||||
48
docs/ru/operations/system-tables/crash-log.md
Normal file
48
docs/ru/operations/system-tables/crash-log.md
Normal file
@ -0,0 +1,48 @@
|
||||
# system.crash_log {#system-tables_crash_log}
|
||||
|
||||
Содержит информацию о трассировках стека для фатальных ошибок. Таблица не содержится в базе данных по умолчанию, а создается только при возникновении фатальных ошибок.
|
||||
|
||||
Колонки:
|
||||
|
||||
- `event_date` ([Datetime](../../sql-reference/data-types/datetime.md)) — Дата события.
|
||||
- `event_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Время события.
|
||||
- `timestamp_ns` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Время события с наносекундами.
|
||||
- `signal` ([Int32](../../sql-reference/data-types/int-uint.md)) — Номер сигнала, пришедшего в поток.
|
||||
- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Идентификатор треда.
|
||||
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Идентификатор запроса.
|
||||
- `trace` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Трассировка стека в момент ошибки. Представляет собой список физических адресов, по которым расположены вызываемые методы.
|
||||
- `trace_full` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Трассировка стека в момент ошибки. Содержит вызываемые методы.
|
||||
- `version` ([String](../../sql-reference/data-types/string.md)) — Версия сервера ClickHouse.
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Ревизия сборки сервера ClickHouse.
|
||||
- `build_id` ([String](../../sql-reference/data-types/string.md)) — BuildID, сгенерированный компилятором.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.crash_log ORDER BY event_time DESC LIMIT 1;
|
||||
```
|
||||
|
||||
Результат (приведён не полностью):
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
event_date: 2020-10-14
|
||||
event_time: 2020-10-14 15:47:40
|
||||
timestamp_ns: 1602679660271312710
|
||||
signal: 11
|
||||
thread_id: 23624
|
||||
query_id: 428aab7c-8f5c-44e9-9607-d16b44467e69
|
||||
trace: [188531193,...]
|
||||
trace_full: ['3. DB::(anonymous namespace)::FunctionFormatReadableTimeDelta::executeImpl(std::__1::vector<DB::ColumnWithTypeAndName, std::__1::allocator<DB::ColumnWithTypeAndName> >&, std::__1::vector<unsigned long, std::__1::allocator<unsigned long> > const&, unsigned long, unsigned long) const @ 0xb3cc1f9 in /home/username/work/ClickHouse/build/programs/clickhouse',...]
|
||||
version: ClickHouse 20.11.1.1
|
||||
revision: 54442
|
||||
build_id:
|
||||
```
|
||||
|
||||
**См. также**
|
||||
- Системная таблица [trace_log](../../operations/system-tables/trace_log.md)
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system-tables/crash-log)
|
||||
@ -10,12 +10,83 @@ SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [F
|
||||
|
||||
## SHOW DATABASES {#show-databases}
|
||||
|
||||
Выводит список всех баз данных.
|
||||
|
||||
```sql
|
||||
SHOW DATABASES [INTO OUTFILE filename] [FORMAT format]
|
||||
SHOW DATABASES [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
Выводит список всех баз данных.
|
||||
Запрос полностью аналогичен запросу `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`.
|
||||
Этот запрос идентичен запросу:
|
||||
|
||||
```sql
|
||||
SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
### Примеры {#examples}
|
||||
|
||||
Получение списка баз данных, имена которых содержат последовательность символов 'de':
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES LIKE '%de%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name────┐
|
||||
│ default │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Получение списка баз данных, имена которых содержат последовательность символов 'de' независимо от регистра:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES ILIKE '%DE%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name────┐
|
||||
│ default │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
Получение списка баз данных, имена которых не содержат последовательность символов 'de':
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES NOT LIKE '%de%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ _temporary_and_external_tables │
|
||||
│ system │
|
||||
│ test │
|
||||
│ tutorial │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
Получение первых двух строк из списка имен баз данных:
|
||||
|
||||
``` sql
|
||||
SHOW DATABASES LIMIT 2
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ _temporary_and_external_tables │
|
||||
│ default │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Смотрите также {#see-also}
|
||||
|
||||
- [CREATE DATABASE](https://clickhouse.tech/docs/ru/sql-reference/statements/create/database/#query-language-create-database)
|
||||
|
||||
## SHOW PROCESSLIST {#show-processlist}
|
||||
|
||||
@ -38,32 +109,85 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
|
||||
Выводит список таблиц.
|
||||
|
||||
```sql
|
||||
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE '<pattern>' | WHERE expr] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
```
|
||||
|
||||
Если секция `FROM` не используется, то запрос возвращает список таблиц из текущей базы данных.
|
||||
Если условие `FROM` не указано, запрос возвращает список таблиц из текущей базы данных.
|
||||
|
||||
Результат, идентичный тому, что выдаёт запрос `SHOW TABLES` можно получить также запросом следующего вида:
|
||||
Этот запрос идентичен запросу:
|
||||
|
||||
```sql
|
||||
SELECT name FROM system.tables WHERE database = <db> [AND name LIKE <pattern>] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
|
||||
```
|
||||
|
||||
**Пример**
|
||||
### Примеры {#examples}
|
||||
|
||||
Следующий запрос выбирает первые две строки из списка таблиц в базе данных `system`, чьи имена содержат `co`.
|
||||
Получение списка таблиц, имена которых содержат последовательность символов 'user':
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system LIKE '%co%' LIMIT 2
|
||||
SHOW TABLES FROM system LIKE '%user%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name─────────────┐
|
||||
│ user_directories │
|
||||
│ users │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
Получение списка таблиц, имена которых содержат последовательность символов 'user' без учета регистра:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system ILIKE '%USER%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name─────────────┐
|
||||
│ user_directories │
|
||||
│ users │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
Получение списка таблиц, имена которых не содержат символ 's':
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system NOT LIKE '%s%'
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name─────────┐
|
||||
│ metric_log │
|
||||
│ metric_log_0 │
|
||||
│ metric_log_1 │
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
Получение первых двух строк из списка таблиц:
|
||||
|
||||
``` sql
|
||||
SHOW TABLES FROM system LIMIT 2
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─name───────────────────────────┐
|
||||
│ aggregate_function_combinators │
|
||||
│ collations │
|
||||
│ asynchronous_metric_log │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Смотрите также {#see-also}
|
||||
|
||||
- [Create Tables](https://clickhouse.tech/docs/ru/getting-started/tutorial/#create-tables)
|
||||
- [SHOW CREATE TABLE](https://clickhouse.tech/docs/ru/sql-reference/statements/show/#show-create-table)
|
||||
|
||||
## SHOW DICTIONARIES {#show-dictionaries}
|
||||
|
||||
Выводит список [внешних словарей](../../sql-reference/statements/show.md).
|
||||
|
||||
@ -1896,7 +1896,7 @@ private:
|
||||
if (has_vertical_output_suffix)
|
||||
throw Exception("Output format already specified", ErrorCodes::CLIENT_OUTPUT_FORMAT_SPECIFIED);
|
||||
const auto & id = query_with_output->format->as<ASTIdentifier &>();
|
||||
current_format = id.name;
|
||||
current_format = id.name();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -172,7 +172,7 @@ ASTPtr extractOrderBy(const ASTPtr & storage_ast)
|
||||
std::string wrapIdentifiersWithBackticks(const ASTPtr & root)
|
||||
{
|
||||
if (auto identifier = std::dynamic_pointer_cast<ASTIdentifier>(root))
|
||||
return backQuote(identifier->name);
|
||||
return backQuote(identifier->name());
|
||||
|
||||
if (auto function = std::dynamic_pointer_cast<ASTFunction>(root))
|
||||
return function->name + '(' + wrapIdentifiersWithBackticks(function->arguments) + ')';
|
||||
|
||||
@ -57,8 +57,8 @@ LocalServer::LocalServer() = default;
|
||||
|
||||
LocalServer::~LocalServer()
|
||||
{
|
||||
if (context)
|
||||
context->shutdown(); /// required for properly exception handling
|
||||
if (global_context)
|
||||
global_context->shutdown(); /// required for properly exception handling
|
||||
}
|
||||
|
||||
|
||||
@ -95,9 +95,9 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
||||
}
|
||||
}
|
||||
|
||||
void LocalServer::applyCmdSettings()
|
||||
void LocalServer::applyCmdSettings(Context & context)
|
||||
{
|
||||
context->applySettingsChanges(cmd_settings.changes());
|
||||
context.applySettingsChanges(cmd_settings.changes());
|
||||
}
|
||||
|
||||
/// If path is specified and not empty, will try to setup server environment and load existing metadata
|
||||
@ -151,8 +151,12 @@ void LocalServer::tryInitPath()
|
||||
if (path.back() != '/')
|
||||
path += '/';
|
||||
|
||||
context->setPath(path);
|
||||
context->setUserFilesPath(""); // user's files are everywhere
|
||||
global_context->setPath(path);
|
||||
|
||||
global_context->setTemporaryStorage(path + "tmp");
|
||||
global_context->setFlagsPath(path + "flags");
|
||||
|
||||
global_context->setUserFilesPath(""); // user's files are everywhere
|
||||
}
|
||||
|
||||
|
||||
@ -186,9 +190,9 @@ try
|
||||
}
|
||||
|
||||
shared_context = Context::createShared();
|
||||
context = std::make_unique<Context>(Context::createGlobal(shared_context.get()));
|
||||
context->makeGlobalContext();
|
||||
context->setApplicationType(Context::ApplicationType::LOCAL);
|
||||
global_context = std::make_unique<Context>(Context::createGlobal(shared_context.get()));
|
||||
global_context->makeGlobalContext();
|
||||
global_context->setApplicationType(Context::ApplicationType::LOCAL);
|
||||
tryInitPath();
|
||||
|
||||
std::optional<StatusFile> status;
|
||||
@ -210,32 +214,32 @@ try
|
||||
|
||||
/// Maybe useless
|
||||
if (config().has("macros"))
|
||||
context->setMacros(std::make_unique<Macros>(config(), "macros", log));
|
||||
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
|
||||
|
||||
/// Skip networking
|
||||
|
||||
/// Sets external authenticators config (LDAP).
|
||||
context->setExternalAuthenticatorsConfig(config());
|
||||
global_context->setExternalAuthenticatorsConfig(config());
|
||||
|
||||
setupUsers();
|
||||
|
||||
/// Limit on total number of concurrently executing queries.
|
||||
/// There is no need for concurrent queries, override max_concurrent_queries.
|
||||
context->getProcessList().setMaxSize(0);
|
||||
global_context->getProcessList().setMaxSize(0);
|
||||
|
||||
/// Size of cache for uncompressed blocks. Zero means disabled.
|
||||
size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
|
||||
if (uncompressed_cache_size)
|
||||
context->setUncompressedCache(uncompressed_cache_size);
|
||||
global_context->setUncompressedCache(uncompressed_cache_size);
|
||||
|
||||
/// Size of cache for marks (index of MergeTree family of tables). It is necessary.
|
||||
/// Specify default value for mark_cache_size explicitly!
|
||||
size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
|
||||
if (mark_cache_size)
|
||||
context->setMarkCache(mark_cache_size);
|
||||
global_context->setMarkCache(mark_cache_size);
|
||||
|
||||
/// Load global settings from default_profile and system_profile.
|
||||
context->setDefaultProfiles(config());
|
||||
global_context->setDefaultProfiles(config());
|
||||
|
||||
/** Init dummy default DB
|
||||
* NOTE: We force using isolated default database to avoid conflicts with default database from server environment
|
||||
@ -243,34 +247,34 @@ try
|
||||
* if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
|
||||
*/
|
||||
std::string default_database = config().getString("default_database", "_local");
|
||||
DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *context));
|
||||
context->setCurrentDatabase(default_database);
|
||||
applyCmdOptions();
|
||||
DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *global_context));
|
||||
global_context->setCurrentDatabase(default_database);
|
||||
applyCmdOptions(*global_context);
|
||||
|
||||
String path = context->getPath();
|
||||
String path = global_context->getPath();
|
||||
if (!path.empty())
|
||||
{
|
||||
/// Lock path directory before read
|
||||
status.emplace(context->getPath() + "status", StatusFile::write_full_info);
|
||||
status.emplace(global_context->getPath() + "status", StatusFile::write_full_info);
|
||||
|
||||
LOG_DEBUG(log, "Loading metadata from {}", path);
|
||||
Poco::File(path + "data/").createDirectories();
|
||||
Poco::File(path + "metadata/").createDirectories();
|
||||
loadMetadataSystem(*context);
|
||||
attachSystemTables(*context);
|
||||
loadMetadata(*context);
|
||||
loadMetadataSystem(*global_context);
|
||||
attachSystemTables(*global_context);
|
||||
loadMetadata(*global_context);
|
||||
DatabaseCatalog::instance().loadDatabases();
|
||||
LOG_DEBUG(log, "Loaded metadata.");
|
||||
}
|
||||
else
|
||||
{
|
||||
attachSystemTables(*context);
|
||||
attachSystemTables(*global_context);
|
||||
}
|
||||
|
||||
processQueries();
|
||||
|
||||
context->shutdown();
|
||||
context.reset();
|
||||
global_context->shutdown();
|
||||
global_context.reset();
|
||||
|
||||
status.reset();
|
||||
cleanup();
|
||||
@ -323,7 +327,7 @@ void LocalServer::processQueries()
|
||||
String initial_create_query = getInitialCreateTableQuery();
|
||||
String queries_str = initial_create_query + config().getRawString("query");
|
||||
|
||||
const auto & settings = context->getSettingsRef();
|
||||
const auto & settings = global_context->getSettingsRef();
|
||||
|
||||
std::vector<String> queries;
|
||||
auto parse_res = splitMultipartQuery(queries_str, queries, settings.max_query_size, settings.max_parser_depth);
|
||||
@ -331,15 +335,19 @@ void LocalServer::processQueries()
|
||||
if (!parse_res.second)
|
||||
throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR);
|
||||
|
||||
context->makeSessionContext();
|
||||
context->makeQueryContext();
|
||||
/// we can't mutate global global_context (can lead to races, as it was already passed to some background threads)
|
||||
/// so we can't reuse it safely as a query context and need a copy here
|
||||
auto context = Context(*global_context);
|
||||
|
||||
context->setUser("default", "", Poco::Net::SocketAddress{});
|
||||
context->setCurrentQueryId("");
|
||||
applyCmdSettings();
|
||||
context.makeSessionContext();
|
||||
context.makeQueryContext();
|
||||
|
||||
context.setUser("default", "", Poco::Net::SocketAddress{});
|
||||
context.setCurrentQueryId("");
|
||||
applyCmdSettings(context);
|
||||
|
||||
/// Use the same query_id (and thread group) for all queries
|
||||
CurrentThread::QueryScope query_scope_holder(*context);
|
||||
CurrentThread::QueryScope query_scope_holder(context);
|
||||
|
||||
bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
|
||||
std::exception_ptr exception;
|
||||
@ -358,7 +366,7 @@ void LocalServer::processQueries()
|
||||
|
||||
try
|
||||
{
|
||||
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {});
|
||||
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {});
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -423,7 +431,7 @@ void LocalServer::setupUsers()
|
||||
}
|
||||
|
||||
if (users_config)
|
||||
context->setUsersConfig(users_config);
|
||||
global_context->setUsersConfig(users_config);
|
||||
else
|
||||
throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
|
||||
}
|
||||
@ -577,10 +585,10 @@ void LocalServer::init(int argc, char ** argv)
|
||||
argsToConfig(arguments, config(), 100);
|
||||
}
|
||||
|
||||
void LocalServer::applyCmdOptions()
|
||||
void LocalServer::applyCmdOptions(Context & context)
|
||||
{
|
||||
context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
|
||||
applyCmdSettings();
|
||||
context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
|
||||
applyCmdSettings(context);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -36,15 +36,15 @@ private:
|
||||
std::string getInitialCreateTableQuery();
|
||||
|
||||
void tryInitPath();
|
||||
void applyCmdOptions();
|
||||
void applyCmdSettings();
|
||||
void applyCmdOptions(Context & context);
|
||||
void applyCmdSettings(Context & context);
|
||||
void processQueries();
|
||||
void setupUsers();
|
||||
void cleanup();
|
||||
|
||||
protected:
|
||||
SharedContextHolder shared_context;
|
||||
std::unique_ptr<Context> context;
|
||||
std::unique_ptr<Context> global_context;
|
||||
|
||||
/// Settings specified via command line args
|
||||
Settings cmd_settings;
|
||||
|
||||
@ -258,7 +258,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
Poco::Logger * log = &logger();
|
||||
UseSSL use_ssl;
|
||||
|
||||
ThreadStatus thread_status;
|
||||
MainThreadStatus::getInstance();
|
||||
|
||||
registerFunctions();
|
||||
registerAggregateFunctions();
|
||||
|
||||
@ -392,6 +392,22 @@
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster_two_shards>
|
||||
<test_cluster_two_shards_internal_replication>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>127.0.0.1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>127.0.0.2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster_two_shards_internal_replication>
|
||||
<test_shard_localhost_secure>
|
||||
<shard>
|
||||
<replica>
|
||||
|
||||
@ -137,7 +137,6 @@ AccessControlManager::AccessControlManager()
|
||||
|
||||
AccessControlManager::~AccessControlManager() = default;
|
||||
|
||||
|
||||
void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguration & users_config_)
|
||||
{
|
||||
auto storages = getStoragesPtr();
|
||||
@ -163,6 +162,7 @@ void AccessControlManager::addUsersConfigStorage(const String & storage_name_, c
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function);
|
||||
new_storage->setConfig(users_config_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
}
|
||||
|
||||
void AccessControlManager::addUsersConfigStorage(
|
||||
@ -195,6 +195,7 @@ void AccessControlManager::addUsersConfigStorage(
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function);
|
||||
new_storage->load(users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
}
|
||||
|
||||
void AccessControlManager::reloadUsersConfigs()
|
||||
@ -238,7 +239,9 @@ void AccessControlManager::addDiskStorage(const String & storage_name_, const St
|
||||
}
|
||||
}
|
||||
}
|
||||
addStorage(std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_));
|
||||
auto new_storage = std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
}
|
||||
|
||||
|
||||
@ -250,13 +253,17 @@ void AccessControlManager::addMemoryStorage(const String & storage_name_)
|
||||
if (auto memory_storage = typeid_cast<std::shared_ptr<MemoryAccessStorage>>(storage))
|
||||
return;
|
||||
}
|
||||
addStorage(std::make_shared<MemoryAccessStorage>(storage_name_));
|
||||
auto new_storage = std::make_shared<MemoryAccessStorage>(storage_name_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName());
|
||||
}
|
||||
|
||||
|
||||
void AccessControlManager::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_)
|
||||
{
|
||||
addStorage(std::make_shared<LDAPAccessStorage>(storage_name_, this, config_, prefix_));
|
||||
auto new_storage = std::make_shared<LDAPAccessStorage>(storage_name_, this, config_, prefix_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -29,6 +29,12 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl
|
||||
}
|
||||
|
||||
|
||||
String LDAPAccessStorage::getLDAPServerName() const
|
||||
{
|
||||
return ldap_server;
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix)
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
|
||||
@ -32,6 +32,8 @@ public:
|
||||
explicit LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
virtual ~LDAPAccessStorage() override = default;
|
||||
|
||||
String getLDAPServerName() const;
|
||||
|
||||
public: // IAccessStorage implementations.
|
||||
virtual const char * getStorageType() const override;
|
||||
virtual String getStorageParamsJSON() const override;
|
||||
|
||||
@ -13,7 +13,6 @@
|
||||
// this one: https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/roaring.h
|
||||
#include <roaring/roaring.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/**
|
||||
@ -599,128 +598,6 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
/// To read and write the DB Buffer directly, migrate code from CRoaring
|
||||
void db_roaring_bitmap_add_many(DB::ReadBuffer & db_buf, roaring_bitmap_t * r, size_t n_args)
|
||||
{
|
||||
void * container = nullptr; // hold value of last container touched
|
||||
uint8_t typecode = 0; // typecode of last container touched
|
||||
uint32_t prev = 0; // previous valued inserted
|
||||
size_t i = 0; // index of value
|
||||
int containerindex = 0;
|
||||
if (n_args == 0)
|
||||
return;
|
||||
uint32_t val;
|
||||
readBinary(val, db_buf);
|
||||
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
|
||||
prev = val;
|
||||
++i;
|
||||
for (; i < n_args; ++i)
|
||||
{
|
||||
readBinary(val, db_buf);
|
||||
if (((prev ^ val) >> 16) == 0)
|
||||
{ // no need to seek the container, it is at hand
|
||||
// because we already have the container at hand, we can do the
|
||||
// insertion
|
||||
// automatically, bypassing the roaring_bitmap_add call
|
||||
uint8_t newtypecode = typecode;
|
||||
void * container2 = container_add(container, val & 0xFFFF, typecode, &newtypecode);
|
||||
// rare instance when we need to
|
||||
if (container2 != container)
|
||||
{
|
||||
// change the container type
|
||||
container_free(container, typecode);
|
||||
ra_set_container_at_index(&r->high_low_container, containerindex, container2, newtypecode);
|
||||
typecode = newtypecode;
|
||||
container = container2;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
|
||||
}
|
||||
prev = val;
|
||||
}
|
||||
}
|
||||
|
||||
void db_ra_to_uint32_array(DB::WriteBuffer & db_buf, roaring_array_t * ra) const
|
||||
{
|
||||
size_t ctr = 0;
|
||||
for (Int32 i = 0; i < ra->size; ++i)
|
||||
{
|
||||
Int32 num_added = db_container_to_uint32_array(db_buf, ra->containers[i], ra->typecodes[i], (static_cast<UInt32>(ra->keys[i])) << 16);
|
||||
ctr += num_added;
|
||||
}
|
||||
}
|
||||
|
||||
UInt32 db_container_to_uint32_array(DB::WriteBuffer & db_buf, const void * container, uint8_t typecode, UInt32 base) const
|
||||
{
|
||||
container = container_unwrap_shared(container, &typecode);
|
||||
switch (typecode)
|
||||
{
|
||||
case BITSET_CONTAINER_TYPE_CODE:
|
||||
return db_bitset_container_to_uint32_array(db_buf, static_cast<const bitset_container_t *>(container), base);
|
||||
case ARRAY_CONTAINER_TYPE_CODE:
|
||||
return db_array_container_to_uint32_array(db_buf, static_cast<const array_container_t *>(container), base);
|
||||
case RUN_CONTAINER_TYPE_CODE:
|
||||
return db_run_container_to_uint32_array(db_buf, static_cast<const run_container_t *>(container), base);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
UInt32 db_bitset_container_to_uint32_array(DB::WriteBuffer & db_buf, const bitset_container_t * cont, UInt32 base) const
|
||||
{
|
||||
return static_cast<UInt32>(db_bitset_extract_setbits(db_buf, cont->array, BITSET_CONTAINER_SIZE_IN_WORDS, base));
|
||||
}
|
||||
|
||||
size_t db_bitset_extract_setbits(DB::WriteBuffer & db_buf, UInt64 * bitset, size_t length, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (size_t i = 0; i < length; ++i)
|
||||
{
|
||||
UInt64 w = bitset[i];
|
||||
while (w != 0)
|
||||
{
|
||||
UInt64 t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail)
|
||||
UInt32 r = __builtin_ctzll(w); // on x64, should compile to TZCNT
|
||||
UInt32 val = r + base;
|
||||
writePODBinary(val, db_buf);
|
||||
outpos++;
|
||||
w ^= t;
|
||||
}
|
||||
base += 64;
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
|
||||
int db_array_container_to_uint32_array(DB::WriteBuffer & db_buf, const array_container_t * cont, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (Int32 i = 0; i < cont->cardinality; ++i)
|
||||
{
|
||||
const UInt32 val = base + cont->array[i];
|
||||
writePODBinary(val, db_buf);
|
||||
outpos++;
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
|
||||
int db_run_container_to_uint32_array(DB::WriteBuffer & db_buf, const run_container_t * cont, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (Int32 i = 0; i < cont->n_runs; ++i)
|
||||
{
|
||||
UInt32 run_start = base + cont->runs[i].value;
|
||||
UInt16 le = cont->runs[i].length;
|
||||
for (Int32 j = 0; j <= le; ++j)
|
||||
{
|
||||
UInt32 val = run_start + j;
|
||||
writePODBinary(val, db_buf);
|
||||
outpos++;
|
||||
}
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
|
||||
52
src/AggregateFunctions/AggregateFunctionStudentTTest.cpp
Normal file
52
src/AggregateFunctions/AggregateFunctionStudentTTest.cpp
Normal file
@ -0,0 +1,52 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionStudentTTest.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include "registerAggregateFunctions.h"
|
||||
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
|
||||
|
||||
// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse)
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{
|
||||
assertBinary(name, argument_types);
|
||||
assertNoParameters(name, parameters);
|
||||
|
||||
AggregateFunctionPtr res;
|
||||
|
||||
if (isDecimal(argument_types[0]) || isDecimal(argument_types[1]))
|
||||
{
|
||||
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
else
|
||||
{
|
||||
res.reset(createWithTwoNumericTypes<AggregateFunctionStudentTTest>(*argument_types[0], *argument_types[1], argument_types));
|
||||
}
|
||||
|
||||
if (!res)
|
||||
{
|
||||
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive);
|
||||
}
|
||||
}
|
||||
253
src/AggregateFunctions/AggregateFunctionStudentTTest.h
Normal file
253
src/AggregateFunctions/AggregateFunctionStudentTTest.h
Normal file
@ -0,0 +1,253 @@
|
||||
#pragma once
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <limits>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename X = Float64, typename Y = Float64>
|
||||
struct AggregateFunctionStudentTTestData final
|
||||
{
|
||||
size_t size_x = 0;
|
||||
size_t size_y = 0;
|
||||
X sum_x = static_cast<X>(0);
|
||||
Y sum_y = static_cast<Y>(0);
|
||||
X square_sum_x = static_cast<X>(0);
|
||||
Y square_sum_y = static_cast<Y>(0);
|
||||
Float64 mean_x = static_cast<Float64>(0);
|
||||
Float64 mean_y = static_cast<Float64>(0);
|
||||
|
||||
void add(X x, Y y)
|
||||
{
|
||||
sum_x += x;
|
||||
sum_y += y;
|
||||
size_x++;
|
||||
size_y++;
|
||||
mean_x = static_cast<Float64>(sum_x) / size_x;
|
||||
mean_y = static_cast<Float64>(sum_y) / size_y;
|
||||
square_sum_x += x * x;
|
||||
square_sum_y += y * y;
|
||||
}
|
||||
|
||||
void merge(const AggregateFunctionStudentTTestData &other)
|
||||
{
|
||||
sum_x += other.sum_x;
|
||||
sum_y += other.sum_y;
|
||||
size_x += other.size_x;
|
||||
size_y += other.size_y;
|
||||
mean_x = static_cast<Float64>(sum_x) / size_x;
|
||||
mean_y = static_cast<Float64>(sum_y) / size_y;
|
||||
square_sum_x += other.square_sum_x;
|
||||
square_sum_y += other.square_sum_y;
|
||||
}
|
||||
|
||||
void serialize(WriteBuffer &buf) const
|
||||
{
|
||||
writeBinary(mean_x, buf);
|
||||
writeBinary(mean_y, buf);
|
||||
writeBinary(sum_x, buf);
|
||||
writeBinary(sum_y, buf);
|
||||
writeBinary(square_sum_x, buf);
|
||||
writeBinary(square_sum_y, buf);
|
||||
writeBinary(size_x, buf);
|
||||
writeBinary(size_y, buf);
|
||||
}
|
||||
|
||||
void deserialize(ReadBuffer &buf)
|
||||
{
|
||||
readBinary(mean_x, buf);
|
||||
readBinary(mean_y, buf);
|
||||
readBinary(sum_x, buf);
|
||||
readBinary(sum_y, buf);
|
||||
readBinary(square_sum_x, buf);
|
||||
readBinary(square_sum_y, buf);
|
||||
readBinary(size_x, buf);
|
||||
readBinary(size_y, buf);
|
||||
}
|
||||
|
||||
size_t getSizeY() const
|
||||
{
|
||||
return size_y;
|
||||
}
|
||||
|
||||
size_t getSizeX() const
|
||||
{
|
||||
return size_x;
|
||||
}
|
||||
|
||||
Float64 getSSquared() const
|
||||
{
|
||||
/// The original formulae looks like
|
||||
/// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2}
|
||||
/// But we made some mathematical transformations not to store original sequences.
|
||||
/// Also we dropped sqrt, because later it will be squared later.
|
||||
const Float64 all_x = square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x;
|
||||
const Float64 all_y = square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y;
|
||||
return static_cast<Float64>(all_x + all_y) / (size_x + size_y - 2);
|
||||
}
|
||||
|
||||
|
||||
Float64 getTStatisticSquared() const
|
||||
{
|
||||
return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared();
|
||||
}
|
||||
|
||||
Float64 getTStatistic() const
|
||||
{
|
||||
return (mean_x - mean_y) / std::sqrt(getStandartErrorSquared());
|
||||
}
|
||||
|
||||
Float64 getStandartErrorSquared() const
|
||||
{
|
||||
if (size_x == 0 || size_y == 0)
|
||||
throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
return getSSquared() * (1.0 / static_cast<Float64>(size_x) + 1.0 / static_cast<Float64>(size_y));
|
||||
}
|
||||
|
||||
Float64 getDegreesOfFreedom() const
|
||||
{
|
||||
return static_cast<Float64>(size_x + size_y - 2);
|
||||
}
|
||||
|
||||
static Float64 integrateSimpson(Float64 a, Float64 b, std::function<Float64(Float64)> func)
|
||||
{
|
||||
const size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b)));
|
||||
const long double h = (b - a) / iterations;
|
||||
Float64 sum_odds = 0.0;
|
||||
for (size_t i = 1; i < iterations; i += 2)
|
||||
sum_odds += func(a + i * h);
|
||||
Float64 sum_evens = 0.0;
|
||||
for (size_t i = 2; i < iterations; i += 2)
|
||||
sum_evens += func(a + i * h);
|
||||
return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3;
|
||||
}
|
||||
|
||||
Float64 getPValue() const
|
||||
{
|
||||
const Float64 v = getDegreesOfFreedom();
|
||||
const Float64 t = getTStatisticSquared();
|
||||
auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); };
|
||||
Float64 numenator = integrateSimpson(0, v / (t + v), f);
|
||||
Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5));
|
||||
return numenator / denominator;
|
||||
}
|
||||
|
||||
std::pair<Float64, Float64> getResult() const
|
||||
{
|
||||
return std::make_pair(getTStatistic(), getPValue());
|
||||
}
|
||||
};
|
||||
|
||||
/// Returns tuple of (t-statistic, p-value)
|
||||
/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf
|
||||
template <typename X = Float64, typename Y = Float64>
|
||||
class AggregateFunctionStudentTTest :
|
||||
public IAggregateFunctionDataHelper<AggregateFunctionStudentTTestData<X, Y>,AggregateFunctionStudentTTest<X, Y>>
|
||||
{
|
||||
|
||||
public:
|
||||
AggregateFunctionStudentTTest(const DataTypes & arguments)
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionStudentTTestData<X, Y>, AggregateFunctionStudentTTest<X, Y>> ({arguments}, {})
|
||||
{}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return "studentTTest";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
DataTypes types
|
||||
{
|
||||
std::make_shared<DataTypeNumber<Float64>>(),
|
||||
std::make_shared<DataTypeNumber<Float64>>(),
|
||||
};
|
||||
|
||||
Strings names
|
||||
{
|
||||
"t-statistic",
|
||||
"p-value"
|
||||
};
|
||||
|
||||
return std::make_shared<DataTypeTuple>(
|
||||
std::move(types),
|
||||
std::move(names)
|
||||
);
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
auto col_x = assert_cast<const ColumnVector<X> *>(columns[0]);
|
||||
auto col_y = assert_cast<const ColumnVector<Y> *>(columns[1]);
|
||||
|
||||
X x = col_x->getData()[row_num];
|
||||
Y y = col_y->getData()[row_num];
|
||||
|
||||
this->data(place).add(x, y);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
this->data(place).merge(this->data(rhs));
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).serialize(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).deserialize(buf);
|
||||
}
|
||||
|
||||
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override
|
||||
{
|
||||
size_t size_x = this->data(place).getSizeX();
|
||||
size_t size_y = this->data(place).getSizeY();
|
||||
|
||||
if (size_x < 2 || size_y < 2)
|
||||
{
|
||||
throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
Float64 t_statistic = 0.0;
|
||||
Float64 p_value = 0.0;
|
||||
std::tie(t_statistic, p_value) = this->data(place).getResult();
|
||||
|
||||
/// Because p-value is a probability.
|
||||
p_value = std::min(1.0, std::max(0.0, p_value));
|
||||
|
||||
auto & column_tuple = assert_cast<ColumnTuple &>(to);
|
||||
auto & column_stat = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(0));
|
||||
auto & column_value = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(1));
|
||||
|
||||
column_stat.getData().push_back(t_statistic);
|
||||
column_value.getData().push_back(p_value);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
};
|
||||
@ -92,7 +92,7 @@ struct AggregateFunctionTimeSeriesGroupSumData
|
||||
it_ss->second.add(t, v);
|
||||
}
|
||||
if (result.size() > 0 && t < result.back().first)
|
||||
throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc!!!", ErrorCodes::LOGICAL_ERROR};
|
||||
throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc.", ErrorCodes::LOGICAL_ERROR};
|
||||
if (result.size() > 0 && t == result.back().first)
|
||||
{
|
||||
//do not add new point
|
||||
|
||||
49
src/AggregateFunctions/AggregateFunctionWelchTTest.cpp
Normal file
49
src/AggregateFunctions/AggregateFunctionWelchTTest.cpp
Normal file
@ -0,0 +1,49 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionWelchTTest.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include "registerAggregateFunctions.h"
|
||||
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{
|
||||
assertBinary(name, argument_types);
|
||||
assertNoParameters(name, parameters);
|
||||
|
||||
AggregateFunctionPtr res;
|
||||
|
||||
if (isDecimal(argument_types[0]) || isDecimal(argument_types[1]))
|
||||
{
|
||||
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
else
|
||||
{
|
||||
res.reset(createWithTwoNumericTypes<AggregateFunctionWelchTTest>(*argument_types[0], *argument_types[1], argument_types));
|
||||
}
|
||||
|
||||
if (!res)
|
||||
{
|
||||
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive);
|
||||
}
|
||||
}
|
||||
264
src/AggregateFunctions/AggregateFunctionWelchTTest.h
Normal file
264
src/AggregateFunctions/AggregateFunctionWelchTTest.h
Normal file
@ -0,0 +1,264 @@
|
||||
#pragma once
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Core/Types.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <limits>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename X = Float64, typename Y = Float64>
|
||||
struct AggregateFunctionWelchTTestData final
|
||||
{
|
||||
size_t size_x = 0;
|
||||
size_t size_y = 0;
|
||||
X sum_x = static_cast<X>(0);
|
||||
Y sum_y = static_cast<Y>(0);
|
||||
X square_sum_x = static_cast<X>(0);
|
||||
Y square_sum_y = static_cast<Y>(0);
|
||||
Float64 mean_x = static_cast<Float64>(0);
|
||||
Float64 mean_y = static_cast<Float64>(0);
|
||||
|
||||
void add(X x, Y y)
|
||||
{
|
||||
sum_x += x;
|
||||
sum_y += y;
|
||||
size_x++;
|
||||
size_y++;
|
||||
mean_x = static_cast<Float64>(sum_x) / size_x;
|
||||
mean_y = static_cast<Float64>(sum_y) / size_y;
|
||||
square_sum_x += x * x;
|
||||
square_sum_y += y * y;
|
||||
}
|
||||
|
||||
void merge(const AggregateFunctionWelchTTestData &other)
|
||||
{
|
||||
sum_x += other.sum_x;
|
||||
sum_y += other.sum_y;
|
||||
size_x += other.size_x;
|
||||
size_y += other.size_y;
|
||||
mean_x = static_cast<Float64>(sum_x) / size_x;
|
||||
mean_y = static_cast<Float64>(sum_y) / size_y;
|
||||
square_sum_x += other.square_sum_x;
|
||||
square_sum_y += other.square_sum_y;
|
||||
}
|
||||
|
||||
void serialize(WriteBuffer &buf) const
|
||||
{
|
||||
writeBinary(mean_x, buf);
|
||||
writeBinary(mean_y, buf);
|
||||
writeBinary(sum_x, buf);
|
||||
writeBinary(sum_y, buf);
|
||||
writeBinary(square_sum_x, buf);
|
||||
writeBinary(square_sum_y, buf);
|
||||
writeBinary(size_x, buf);
|
||||
writeBinary(size_y, buf);
|
||||
}
|
||||
|
||||
void deserialize(ReadBuffer &buf)
|
||||
{
|
||||
readBinary(mean_x, buf);
|
||||
readBinary(mean_y, buf);
|
||||
readBinary(sum_x, buf);
|
||||
readBinary(sum_y, buf);
|
||||
readBinary(square_sum_x, buf);
|
||||
readBinary(square_sum_y, buf);
|
||||
readBinary(size_x, buf);
|
||||
readBinary(size_y, buf);
|
||||
}
|
||||
|
||||
size_t getSizeY() const
|
||||
{
|
||||
return size_y;
|
||||
}
|
||||
|
||||
size_t getSizeX() const
|
||||
{
|
||||
return size_x;
|
||||
}
|
||||
|
||||
Float64 getSxSquared() const
|
||||
{
|
||||
/// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2}
|
||||
/// But we made some mathematical transformations not to store original sequences.
|
||||
/// Also we dropped sqrt, because later it will be squared later.
|
||||
return static_cast<Float64>(square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x) / (size_x - 1);
|
||||
}
|
||||
|
||||
Float64 getSySquared() const
|
||||
{
|
||||
/// The original formulae looks like \frac{1}{size_y - 1} \sum_{i = 1}^{size_y}{(y_i - \bar{y}) ^ 2}
|
||||
/// But we made some mathematical transformations not to store original sequences.
|
||||
/// Also we dropped sqrt, because later it will be squared later.
|
||||
return static_cast<Float64>(square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y) / (size_y - 1);
|
||||
}
|
||||
|
||||
Float64 getTStatisticSquared() const
|
||||
{
|
||||
if (size_x == 0 || size_y == 0)
|
||||
{
|
||||
throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
return std::pow(mean_x - mean_y, 2) / (getSxSquared() / size_x + getSySquared() / size_y);
|
||||
}
|
||||
|
||||
Float64 getTStatistic() const
|
||||
{
|
||||
if (size_x == 0 || size_y == 0)
|
||||
{
|
||||
throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
return (mean_x - mean_y) / std::sqrt(getSxSquared() / size_x + getSySquared() / size_y);
|
||||
}
|
||||
|
||||
Float64 getDegreesOfFreedom() const
|
||||
{
|
||||
auto sx = getSxSquared();
|
||||
auto sy = getSySquared();
|
||||
Float64 numerator = std::pow(sx / size_x + sy / size_y, 2);
|
||||
Float64 denominator_first = std::pow(sx, 2) / (std::pow(size_x, 2) * (size_x - 1));
|
||||
Float64 denominator_second = std::pow(sy, 2) / (std::pow(size_y, 2) * (size_y - 1));
|
||||
return numerator / (denominator_first + denominator_second);
|
||||
}
|
||||
|
||||
static Float64 integrateSimpson(Float64 a, Float64 b, std::function<Float64(Float64)> func)
|
||||
{
|
||||
size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b)));
|
||||
double h = (b - a) / iterations;
|
||||
Float64 sum_odds = 0.0;
|
||||
for (size_t i = 1; i < iterations; i += 2)
|
||||
sum_odds += func(a + i * h);
|
||||
Float64 sum_evens = 0.0;
|
||||
for (size_t i = 2; i < iterations; i += 2)
|
||||
sum_evens += func(a + i * h);
|
||||
return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3;
|
||||
}
|
||||
|
||||
Float64 getPValue() const
|
||||
{
|
||||
const Float64 v = getDegreesOfFreedom();
|
||||
const Float64 t = getTStatisticSquared();
|
||||
auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); };
|
||||
Float64 numenator = integrateSimpson(0, v / (t + v), f);
|
||||
Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5));
|
||||
return numenator / denominator;
|
||||
}
|
||||
|
||||
std::pair<Float64, Float64> getResult() const
|
||||
{
|
||||
return std::make_pair(getTStatistic(), getPValue());
|
||||
}
|
||||
};
|
||||
|
||||
/// Returns tuple of (t-statistic, p-value)
|
||||
/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf
|
||||
template <typename X = Float64, typename Y = Float64>
|
||||
class AggregateFunctionWelchTTest :
|
||||
public IAggregateFunctionDataHelper<AggregateFunctionWelchTTestData<X, Y>,AggregateFunctionWelchTTest<X, Y>>
|
||||
{
|
||||
|
||||
public:
|
||||
AggregateFunctionWelchTTest(const DataTypes & arguments)
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionWelchTTestData<X, Y>, AggregateFunctionWelchTTest<X, Y>> ({arguments}, {})
|
||||
{}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return "welchTTest";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
DataTypes types
|
||||
{
|
||||
std::make_shared<DataTypeNumber<Float64>>(),
|
||||
std::make_shared<DataTypeNumber<Float64>>(),
|
||||
};
|
||||
|
||||
Strings names
|
||||
{
|
||||
"t-statistic",
|
||||
"p-value"
|
||||
};
|
||||
|
||||
return std::make_shared<DataTypeTuple>(
|
||||
std::move(types),
|
||||
std::move(names)
|
||||
);
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
auto col_x = assert_cast<const ColumnVector<X> *>(columns[0]);
|
||||
auto col_y = assert_cast<const ColumnVector<Y> *>(columns[1]);
|
||||
|
||||
X x = col_x->getData()[row_num];
|
||||
Y y = col_y->getData()[row_num];
|
||||
|
||||
this->data(place).add(x, y);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
this->data(place).merge(this->data(rhs));
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).serialize(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).deserialize(buf);
|
||||
}
|
||||
|
||||
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override
|
||||
{
|
||||
size_t size_x = this->data(place).getSizeX();
|
||||
size_t size_y = this->data(place).getSizeY();
|
||||
|
||||
if (size_x < 2 || size_y < 2)
|
||||
{
|
||||
throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
Float64 t_statistic = 0.0;
|
||||
Float64 p_value = 0.0;
|
||||
std::tie(t_statistic, p_value) = this->data(place).getResult();
|
||||
|
||||
/// Because p-value is a probability.
|
||||
p_value = std::min(1.0, std::max(0.0, p_value));
|
||||
|
||||
auto & column_tuple = assert_cast<ColumnTuple &>(to);
|
||||
auto & column_stat = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(0));
|
||||
auto & column_value = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(1));
|
||||
|
||||
column_stat.getData().push_back(t_statistic);
|
||||
column_value.getData().push_back(p_value);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
};
|
||||
@ -45,6 +45,8 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionMoving(factory);
|
||||
registerAggregateFunctionCategoricalIV(factory);
|
||||
registerAggregateFunctionAggThrow(factory);
|
||||
registerAggregateFunctionWelchTTest(factory);
|
||||
registerAggregateFunctionStudentTTest(factory);
|
||||
registerAggregateFunctionRankCorrelation(factory);
|
||||
}
|
||||
|
||||
|
||||
@ -35,6 +35,8 @@ void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &)
|
||||
void registerAggregateFunctionMoving(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionAggThrow(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory &);
|
||||
|
||||
class AggregateFunctionCombinatorFactory;
|
||||
|
||||
@ -42,6 +42,7 @@ SRCS(
|
||||
AggregateFunctionState.cpp
|
||||
AggregateFunctionStatistics.cpp
|
||||
AggregateFunctionStatisticsSimple.cpp
|
||||
AggregateFunctionStudentTTest.cpp
|
||||
AggregateFunctionSum.cpp
|
||||
AggregateFunctionSumMap.cpp
|
||||
AggregateFunctionTimeSeriesGroupSum.cpp
|
||||
@ -49,6 +50,7 @@ SRCS(
|
||||
AggregateFunctionUniqCombined.cpp
|
||||
AggregateFunctionUniq.cpp
|
||||
AggregateFunctionUniqUpTo.cpp
|
||||
AggregateFunctionWelchTTest.cpp
|
||||
AggregateFunctionWindowFunnel.cpp
|
||||
parseAggregateFunctionParameters.cpp
|
||||
registerAggregateFunctions.cpp
|
||||
|
||||
@ -173,14 +173,20 @@ add_object_library(clickhouse_processors_merges Processors/Merges)
|
||||
add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Algorithms)
|
||||
add_object_library(clickhouse_processors_queryplan Processors/QueryPlan)
|
||||
|
||||
set (DBMS_COMMON_LIBRARIES)
|
||||
# libgcc_s does not provide an implementation of an atomics library. Instead,
|
||||
# GCC’s libatomic library can be used to supply these when using libgcc_s.
|
||||
if ((NOT USE_LIBCXX) AND COMPILER_CLANG AND OS_LINUX)
|
||||
list (APPEND DBMS_COMMON_LIBRARIES atomic)
|
||||
endif()
|
||||
|
||||
if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
|
||||
add_library (dbms STATIC ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PRIVATE jemalloc libdivide)
|
||||
target_link_libraries (dbms PRIVATE jemalloc libdivide ${DBMS_COMMON_LIBRARIES})
|
||||
set (all_modules dbms)
|
||||
else()
|
||||
add_library (dbms SHARED ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PUBLIC ${all_modules})
|
||||
target_link_libraries (dbms PUBLIC ${all_modules} ${DBMS_COMMON_LIBRARIES})
|
||||
target_link_libraries (clickhouse_interpreters PRIVATE jemalloc libdivide)
|
||||
list (APPEND all_modules dbms)
|
||||
# force all split libs to be linked
|
||||
|
||||
@ -18,7 +18,7 @@ public:
|
||||
void ALWAYS_INLINE forEachMapped(Func && func)
|
||||
{
|
||||
for (auto i = 0u; i < this->NUM_BUCKETS; ++i)
|
||||
return this->impls[i].forEachMapped(func);
|
||||
this->impls[i].forEachMapped(func);
|
||||
}
|
||||
|
||||
TMapped & ALWAYS_INLINE operator[](const Key & x)
|
||||
|
||||
@ -13,6 +13,24 @@
|
||||
#include <random>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
MemoryTracker * getMemoryTracker()
|
||||
{
|
||||
if (auto * thread_memory_tracker = DB::CurrentThread::getMemoryTracker())
|
||||
return thread_memory_tracker;
|
||||
|
||||
/// Once the main thread is initialized,
|
||||
/// total_memory_tracker is initialized too.
|
||||
/// And can be used, since MainThreadStatus is required for profiling.
|
||||
if (DB::MainThreadStatus::get())
|
||||
return &total_memory_tracker;
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -192,14 +210,15 @@ void MemoryTracker::free(Int64 size)
|
||||
DB::TraceCollector::collect(DB::TraceType::MemorySample, StackTrace(), -size);
|
||||
}
|
||||
|
||||
Int64 accounted_size = size;
|
||||
if (level == VariableContext::Thread)
|
||||
{
|
||||
/// Could become negative if memory allocated in this thread is freed in another one
|
||||
amount.fetch_sub(size, std::memory_order_relaxed);
|
||||
amount.fetch_sub(accounted_size, std::memory_order_relaxed);
|
||||
}
|
||||
else
|
||||
{
|
||||
Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
|
||||
Int64 new_amount = amount.fetch_sub(accounted_size, std::memory_order_relaxed) - accounted_size;
|
||||
|
||||
/** Sometimes, query could free some data, that was allocated outside of query context.
|
||||
* Example: cache eviction.
|
||||
@ -210,7 +229,7 @@ void MemoryTracker::free(Int64 size)
|
||||
if (unlikely(new_amount < 0))
|
||||
{
|
||||
amount.fetch_sub(new_amount);
|
||||
size += new_amount;
|
||||
accounted_size += new_amount;
|
||||
}
|
||||
}
|
||||
|
||||
@ -218,7 +237,7 @@ void MemoryTracker::free(Int64 size)
|
||||
loaded_next->free(size);
|
||||
|
||||
if (metric != CurrentMetrics::end())
|
||||
CurrentMetrics::sub(metric, size);
|
||||
CurrentMetrics::sub(metric, accounted_size);
|
||||
}
|
||||
|
||||
|
||||
@ -270,7 +289,9 @@ namespace CurrentMemoryTracker
|
||||
|
||||
void alloc(Int64 size)
|
||||
{
|
||||
if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker())
|
||||
if (auto * memory_tracker = getMemoryTracker())
|
||||
{
|
||||
if (current_thread)
|
||||
{
|
||||
current_thread->untracked_memory += size;
|
||||
if (current_thread->untracked_memory > current_thread->untracked_memory_limit)
|
||||
@ -282,6 +303,12 @@ namespace CurrentMemoryTracker
|
||||
memory_tracker->alloc(tmp);
|
||||
}
|
||||
}
|
||||
/// total_memory_tracker only, ignore untracked_memory
|
||||
else
|
||||
{
|
||||
memory_tracker->alloc(size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void realloc(Int64 old_size, Int64 new_size)
|
||||
@ -292,7 +319,9 @@ namespace CurrentMemoryTracker
|
||||
|
||||
void free(Int64 size)
|
||||
{
|
||||
if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker())
|
||||
if (auto * memory_tracker = getMemoryTracker())
|
||||
{
|
||||
if (current_thread)
|
||||
{
|
||||
current_thread->untracked_memory -= size;
|
||||
if (current_thread->untracked_memory < -current_thread->untracked_memory_limit)
|
||||
@ -301,5 +330,11 @@ namespace CurrentMemoryTracker
|
||||
current_thread->untracked_memory = 0;
|
||||
}
|
||||
}
|
||||
/// total_memory_tracker only, ignore untracked_memory
|
||||
else
|
||||
{
|
||||
memory_tracker->free(size);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
thread_local ThreadStatus * current_thread = nullptr;
|
||||
thread_local ThreadStatus * main_thread = nullptr;
|
||||
|
||||
|
||||
ThreadStatus::ThreadStatus()
|
||||
@ -115,4 +116,20 @@ void ThreadStatus::onFatalError()
|
||||
fatal_error_callback();
|
||||
}
|
||||
|
||||
ThreadStatus * MainThreadStatus::main_thread = nullptr;
|
||||
MainThreadStatus & MainThreadStatus::getInstance()
|
||||
{
|
||||
static MainThreadStatus thread_status;
|
||||
return thread_status;
|
||||
}
|
||||
MainThreadStatus::MainThreadStatus()
|
||||
: ThreadStatus()
|
||||
{
|
||||
main_thread = current_thread;
|
||||
}
|
||||
MainThreadStatus::~MainThreadStatus()
|
||||
{
|
||||
main_thread = nullptr;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -215,4 +215,22 @@ private:
|
||||
void setupState(const ThreadGroupStatusPtr & thread_group_);
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates ThreadStatus for the main thread.
|
||||
*/
|
||||
class MainThreadStatus : public ThreadStatus
|
||||
{
|
||||
public:
|
||||
static MainThreadStatus & getInstance();
|
||||
static ThreadStatus * get() { return main_thread; }
|
||||
static bool isMainThread() { return main_thread == current_thread; }
|
||||
|
||||
~MainThreadStatus();
|
||||
|
||||
private:
|
||||
MainThreadStatus();
|
||||
|
||||
static ThreadStatus * main_thread;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -66,10 +66,20 @@ void TraceCollector::collect(TraceType trace_type, const StackTrace & stack_trac
|
||||
char buffer[buf_size];
|
||||
WriteBufferFromFileDescriptorDiscardOnFailure out(pipe.fds_rw[1], buf_size, buffer);
|
||||
|
||||
StringRef query_id = CurrentThread::getQueryId();
|
||||
StringRef query_id;
|
||||
UInt64 thread_id;
|
||||
|
||||
if (CurrentThread::isInitialized())
|
||||
{
|
||||
query_id = CurrentThread::getQueryId();
|
||||
query_id.size = std::min(query_id.size, QUERY_ID_MAX_LEN);
|
||||
|
||||
auto thread_id = CurrentThread::get().thread_id;
|
||||
thread_id = CurrentThread::get().thread_id;
|
||||
}
|
||||
else
|
||||
{
|
||||
thread_id = MainThreadStatus::get()->thread_id;
|
||||
}
|
||||
|
||||
writeChar(false, out); /// true if requested to stop the collecting thread.
|
||||
writeStringBinary(query_id, out);
|
||||
|
||||
@ -76,7 +76,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr
|
||||
ASTPtr codec_arguments;
|
||||
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
||||
{
|
||||
codec_family_name = family_name->name;
|
||||
codec_family_name = family_name->name();
|
||||
codec_arguments = {};
|
||||
}
|
||||
else if (const auto * ast_func = inner_codec_ast->as<ASTFunction>())
|
||||
@ -207,7 +207,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IData
|
||||
ASTPtr codec_arguments;
|
||||
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
||||
{
|
||||
codec_family_name = family_name->name;
|
||||
codec_family_name = family_name->name();
|
||||
codec_arguments = {};
|
||||
}
|
||||
else if (const auto * ast_func = inner_codec_ast->as<ASTFunction>())
|
||||
|
||||
@ -5,9 +5,6 @@ target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLU
|
||||
add_executable (field field.cpp)
|
||||
target_link_libraries (field PRIVATE dbms)
|
||||
|
||||
add_executable (move_field move_field.cpp)
|
||||
target_link_libraries (move_field PRIVATE clickhouse_common_io)
|
||||
|
||||
add_executable (string_ref_hash string_ref_hash.cpp)
|
||||
target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)
|
||||
|
||||
|
||||
22
src/Core/tests/gtest_move_field.cpp
Normal file
22
src/Core/tests/gtest_move_field.cpp
Normal file
@ -0,0 +1,22 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <Core/Field.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
GTEST_TEST(Field, Move)
|
||||
{
|
||||
Field f;
|
||||
|
||||
f = Field{String{"Hello, world (1)"}};
|
||||
ASSERT_EQ(f.get<String>(), "Hello, world (1)");
|
||||
f = Field{String{"Hello, world (2)"}};
|
||||
ASSERT_EQ(f.get<String>(), "Hello, world (2)");
|
||||
f = Field{Array{Field{String{"Hello, world (3)"}}}};
|
||||
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (3)");
|
||||
f = String{"Hello, world (4)"};
|
||||
ASSERT_EQ(f.get<String>(), "Hello, world (4)");
|
||||
f = Array{Field{String{"Hello, world (5)"}}};
|
||||
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (5)");
|
||||
f = Array{String{"Hello, world (6)"}};
|
||||
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (6)");
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
#include <iostream>
|
||||
#include <Core/Field.h>
|
||||
|
||||
|
||||
int main(int, char **)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
Field f;
|
||||
|
||||
f = Field{String{"Hello, world"}};
|
||||
std::cerr << f.get<String>() << "\n";
|
||||
f = Field{String{"Hello, world!"}};
|
||||
std::cerr << f.get<String>() << "\n";
|
||||
f = Field{Array{Field{String{"Hello, world!!"}}}};
|
||||
std::cerr << f.get<Array>()[0].get<String>() << "\n";
|
||||
f = String{"Hello, world!!!"};
|
||||
std::cerr << f.get<String>() << "\n";
|
||||
f = Array{Field{String{"Hello, world!!!!"}}};
|
||||
std::cerr << f.get<Array>()[0].get<String>() << "\n";
|
||||
f = Array{String{"Hello, world!!!!!"}};
|
||||
std::cerr << f.get<Array>()[0].get<String>() << "\n";
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -43,7 +43,7 @@ DataTypePtr DataTypeFactory::get(const ASTPtr & ast) const
|
||||
|
||||
if (const auto * ident = ast->as<ASTIdentifier>())
|
||||
{
|
||||
return get(ident->name, {});
|
||||
return get(ident->name(), {});
|
||||
}
|
||||
|
||||
if (const auto * lit = ast->as<ASTLiteral>())
|
||||
|
||||
@ -384,7 +384,7 @@ static DataTypePtr create(const ASTPtr & arguments)
|
||||
throw Exception("String data type family mustn't have more than one argument - size in characters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const auto * argument = arguments->children[0]->as<ASTLiteral>();
|
||||
if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get<UInt64>() == 0)
|
||||
if (!argument || argument->value.getType() != Field::Types::UInt64)
|
||||
throw Exception("String data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
||||
}
|
||||
|
||||
|
||||
@ -329,4 +329,10 @@ const StoragePtr & DatabaseLazyIterator::table() const
|
||||
return current_storage;
|
||||
}
|
||||
|
||||
void DatabaseLazyIterator::reset()
|
||||
{
|
||||
if (current_storage)
|
||||
current_storage.reset();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -122,6 +122,7 @@ public:
|
||||
bool isValid() const override;
|
||||
const String & name() const override;
|
||||
const StoragePtr & table() const override;
|
||||
void reset() override;
|
||||
|
||||
private:
|
||||
const DatabaseLazy & database;
|
||||
|
||||
@ -44,6 +44,8 @@ public:
|
||||
/// (a database with support for lazy tables loading
|
||||
/// - it maintains a list of tables but tables are loaded lazily).
|
||||
virtual const StoragePtr & table() const = 0;
|
||||
/// Reset reference counter to the StoragePtr.
|
||||
virtual void reset() = 0;
|
||||
|
||||
virtual ~IDatabaseTablesIterator() = default;
|
||||
|
||||
@ -93,6 +95,8 @@ public:
|
||||
const String & name() const override { return it->first; }
|
||||
|
||||
const StoragePtr & table() const override { return it->second; }
|
||||
|
||||
void reset() override { it->second.reset(); }
|
||||
};
|
||||
|
||||
/// Copies list of dictionaries and iterates through such snapshot.
|
||||
|
||||
@ -28,6 +28,11 @@ public:
|
||||
return tables.emplace_back(storage);
|
||||
}
|
||||
|
||||
void reset() override
|
||||
{
|
||||
tables.clear();
|
||||
}
|
||||
|
||||
UUID uuid() const override { return nested_iterator->uuid(); }
|
||||
|
||||
DatabaseMaterializeTablesIterator(DatabaseTablesIteratorPtr nested_iterator_, DatabaseMaterializeMySQL * database_)
|
||||
|
||||
@ -172,7 +172,7 @@ Names getPrimaryKeyColumns(const ASTExpressionList * primary_key)
|
||||
for (size_t index = 0; index != children.size(); ++index)
|
||||
{
|
||||
const ASTIdentifier * key_part = children[index]->as<const ASTIdentifier>();
|
||||
result.push_back(key_part->name);
|
||||
result.push_back(key_part->name());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -367,7 +367,7 @@ void buildConfigurationFromFunctionWithKeyValueArguments(
|
||||
|
||||
if (const auto * identifier = pair->second->as<const ASTIdentifier>(); identifier)
|
||||
{
|
||||
AutoPtr<Text> value(doc->createTextNode(identifier->name));
|
||||
AutoPtr<Text> value(doc->createTextNode(identifier->name()));
|
||||
current_xml_element->appendChild(value);
|
||||
}
|
||||
else if (const auto * literal = pair->second->as<const ASTLiteral>(); literal)
|
||||
|
||||
@ -307,6 +307,13 @@ void StoragePolicy::buildVolumeIndices()
|
||||
}
|
||||
}
|
||||
|
||||
bool StoragePolicy::hasAnyVolumeWithDisabledMerges() const
|
||||
{
|
||||
for (const auto & volume : volumes)
|
||||
if (volume->areMergesAvoided())
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
StoragePolicySelector::StoragePolicySelector(
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
|
||||
@ -88,6 +88,9 @@ public:
|
||||
/// Checks if storage policy can be replaced by another one.
|
||||
void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const;
|
||||
|
||||
/// Check if we have any volume with stopped merges
|
||||
bool hasAnyVolumeWithDisabledMerges() const;
|
||||
|
||||
private:
|
||||
Volumes volumes;
|
||||
const String name;
|
||||
|
||||
@ -307,11 +307,6 @@ private:
|
||||
}
|
||||
|
||||
const auto input_value = input_column->getDataAt(r);
|
||||
auto aad_value = StringRef{};
|
||||
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM && !std::is_same_v<std::nullptr_t, std::decay_t<AadColumnType>>)
|
||||
{
|
||||
aad_value = aad_column->getDataAt(r);
|
||||
}
|
||||
|
||||
if constexpr (mode != CipherMode::MySQLCompatibility)
|
||||
{
|
||||
|
||||
@ -82,13 +82,13 @@ public:
|
||||
|
||||
std::shared_ptr<const IDictionaryBase> getDictionary(const String & dictionary_name)
|
||||
{
|
||||
auto dict = std::atomic_load(&dictionary);
|
||||
if (dict)
|
||||
return dict;
|
||||
String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name);
|
||||
dict = external_loader.getDictionary(resolved_name);
|
||||
auto dict = external_loader.getDictionary(resolved_name);
|
||||
if (!access_checked)
|
||||
{
|
||||
context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName());
|
||||
std::atomic_store(&dictionary, dict);
|
||||
access_checked = true;
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
|
||||
@ -122,6 +122,8 @@ private:
|
||||
const Context & context;
|
||||
const ExternalDictionariesLoader & external_loader;
|
||||
mutable std::shared_ptr<const IDictionaryBase> dictionary;
|
||||
/// Access cannot be not granted, since in this case checkAccess() will throw and access_checked will not be updated.
|
||||
std::atomic<bool> access_checked = false;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -290,38 +290,38 @@ private:
|
||||
|
||||
|
||||
/// Apply target function by feeding it "batches" of N columns
|
||||
/// Combining 10 columns per pass is the fastest for large columns sizes.
|
||||
/// For small columns sizes - more columns is faster.
|
||||
/// Combining 8 columns per pass is the fastest method, because it's the maximum when clang vectorizes a loop.
|
||||
template <
|
||||
typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 10>
|
||||
typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 8>
|
||||
struct OperationApplier
|
||||
{
|
||||
template <typename Columns, typename ResultData>
|
||||
static void apply(Columns & in, ResultData & result_data, bool use_result_data_as_input = false)
|
||||
{
|
||||
if (!use_result_data_as_input)
|
||||
doBatchedApply<false>(in, result_data);
|
||||
doBatchedApply<false>(in, result_data.data(), result_data.size());
|
||||
while (!in.empty())
|
||||
doBatchedApply<true>(in, result_data);
|
||||
doBatchedApply<true>(in, result_data.data(), result_data.size());
|
||||
}
|
||||
|
||||
template <bool CarryResult, typename Columns, typename ResultData>
|
||||
static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data)
|
||||
template <bool CarryResult, typename Columns, typename Result>
|
||||
static void NO_INLINE doBatchedApply(Columns & in, Result * __restrict result_data, size_t size)
|
||||
{
|
||||
if (N > in.size())
|
||||
{
|
||||
OperationApplier<Op, OperationApplierImpl, N - 1>
|
||||
::template doBatchedApply<CarryResult>(in, result_data);
|
||||
::template doBatchedApply<CarryResult>(in, result_data, size);
|
||||
return;
|
||||
}
|
||||
|
||||
const OperationApplierImpl<Op, N> operation_applier_impl(in);
|
||||
size_t i = 0;
|
||||
for (auto & res : result_data)
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
if constexpr (CarryResult)
|
||||
res = Op::apply(res, operation_applier_impl.apply(i++));
|
||||
result_data[i] = Op::apply(result_data[i], operation_applier_impl.apply(i));
|
||||
else
|
||||
res = operation_applier_impl.apply(i++);
|
||||
result_data[i] = operation_applier_impl.apply(i);
|
||||
}
|
||||
|
||||
in.erase(in.end() - N, in.end());
|
||||
}
|
||||
@ -332,7 +332,7 @@ template <
|
||||
struct OperationApplier<Op, OperationApplierImpl, 0>
|
||||
{
|
||||
template <bool, typename Columns, typename Result>
|
||||
static void NO_INLINE doBatchedApply(Columns &, Result &)
|
||||
static void NO_INLINE doBatchedApply(Columns &, Result &, size_t)
|
||||
{
|
||||
throw Exception(
|
||||
"OperationApplier<...>::apply(...): not enough arguments to run this method",
|
||||
|
||||
@ -159,7 +159,7 @@ struct NumIfImpl<A, B, NumberTraits::Error>
|
||||
private:
|
||||
[[noreturn]] static void throwError()
|
||||
{
|
||||
throw Exception("Invalid types of arguments 2 and 3 of if", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception("Incompatible types of arguments corresponding to two conditional branches", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
public:
|
||||
template <typename... Args> static ColumnPtr vectorVector(Args &&...) { throwError(); }
|
||||
|
||||
@ -582,7 +582,7 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr & ast,
|
||||
|
||||
/// Special check for WITH statement alias. Add alias action to be able to use this alias.
|
||||
if (identifier.prefer_alias_to_column_name && !identifier.alias.empty())
|
||||
data.addAlias(identifier.name, identifier.alias);
|
||||
data.addAlias(identifier.name(), identifier.alias);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -24,11 +24,12 @@ namespace DB
|
||||
class AddDefaultDatabaseVisitor
|
||||
{
|
||||
public:
|
||||
AddDefaultDatabaseVisitor(const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr)
|
||||
: database_name(database_name_),
|
||||
only_replace_current_database_function(only_replace_current_database_function_),
|
||||
visit_depth(0),
|
||||
ostr(ostr_)
|
||||
explicit AddDefaultDatabaseVisitor(
|
||||
const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr)
|
||||
: database_name(database_name_)
|
||||
, only_replace_current_database_function(only_replace_current_database_function_)
|
||||
, visit_depth(0)
|
||||
, ostr(ostr_)
|
||||
{}
|
||||
|
||||
void visitDDL(ASTPtr & ast) const
|
||||
@ -105,7 +106,7 @@ private:
|
||||
void visit(const ASTIdentifier & identifier, ASTPtr & ast) const
|
||||
{
|
||||
if (!identifier.compound())
|
||||
ast = createTableIdentifier(database_name, identifier.name);
|
||||
ast = createTableIdentifier(database_name, identifier.name());
|
||||
}
|
||||
|
||||
void visit(ASTSubquery & subquery, ASTPtr &) const
|
||||
@ -116,7 +117,7 @@ private:
|
||||
void visit(ASTFunction & function, ASTPtr &) const
|
||||
{
|
||||
bool is_operator_in = false;
|
||||
for (auto name : {"in", "notIn", "globalIn", "globalNotIn"})
|
||||
for (const auto * name : {"in", "notIn", "globalIn", "globalNotIn"})
|
||||
{
|
||||
if (function.name == name)
|
||||
{
|
||||
|
||||
@ -98,33 +98,33 @@ private:
|
||||
if (!IdentifierSemantic::getColumnName(node))
|
||||
return;
|
||||
|
||||
auto split = Nested::splitName(node.name); /// ParsedParams, Key1
|
||||
auto split = Nested::splitName(node.name()); /// ParsedParams, Key1
|
||||
|
||||
if (array_join_alias_to_name.count(node.name))
|
||||
if (array_join_alias_to_name.count(node.name()))
|
||||
{
|
||||
/// ARRAY JOIN was written with an array column. Example: SELECT K1 FROM ... ARRAY JOIN ParsedParams.Key1 AS K1
|
||||
array_join_result_to_source[node.name] = array_join_alias_to_name[node.name]; /// K1 -> ParsedParams.Key1
|
||||
array_join_result_to_source[node.name()] = array_join_alias_to_name[node.name()]; /// K1 -> ParsedParams.Key1
|
||||
}
|
||||
else if (array_join_alias_to_name.count(split.first) && !split.second.empty())
|
||||
{
|
||||
/// ARRAY JOIN was written with a nested table. Example: SELECT PP.KEY1 FROM ... ARRAY JOIN ParsedParams AS PP
|
||||
array_join_result_to_source[node.name] /// PP.Key1 -> ParsedParams.Key1
|
||||
array_join_result_to_source[node.name()] /// PP.Key1 -> ParsedParams.Key1
|
||||
= Nested::concatenateName(array_join_alias_to_name[split.first], split.second);
|
||||
}
|
||||
else if (array_join_name_to_alias.count(node.name))
|
||||
else if (array_join_name_to_alias.count(node.name()))
|
||||
{
|
||||
/** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams.Key1 AS PP.Key1.
|
||||
* That is, the query uses the original array, replicated by itself.
|
||||
*/
|
||||
array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1
|
||||
array_join_name_to_alias[node.name]] = node.name;
|
||||
array_join_name_to_alias[node.name()]] = node.name();
|
||||
}
|
||||
else if (array_join_name_to_alias.count(split.first) && !split.second.empty())
|
||||
{
|
||||
/** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams AS PP.
|
||||
*/
|
||||
array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1
|
||||
Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name;
|
||||
Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -614,14 +614,19 @@ const std::string & Cluster::ShardInfo::pathForInsert(bool prefer_localhost_repl
|
||||
if (!has_internal_replication)
|
||||
throw Exception("internal_replication is not set", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (dir_name_for_internal_replication.empty() || dir_name_for_internal_replication_with_local.empty())
|
||||
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
if (prefer_localhost_replica)
|
||||
{
|
||||
if (dir_name_for_internal_replication.empty())
|
||||
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
|
||||
return dir_name_for_internal_replication;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (dir_name_for_internal_replication_with_local.empty())
|
||||
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
|
||||
return dir_name_for_internal_replication_with_local;
|
||||
}
|
||||
}
|
||||
|
||||
bool Cluster::maybeCrossReplication() const
|
||||
{
|
||||
|
||||
@ -144,11 +144,11 @@ std::pair<size_t, size_t> CollectJoinOnKeysMatcher::getTableNumbers(const ASTPtr
|
||||
|
||||
const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifier * identifier, const Aliases & aliases)
|
||||
{
|
||||
if (identifier->compound())
|
||||
if (identifier->supposedToBeCompound())
|
||||
return identifier;
|
||||
|
||||
UInt32 max_attempts = 100;
|
||||
for (auto it = aliases.find(identifier->name); it != aliases.end();)
|
||||
for (auto it = aliases.find(identifier->name()); it != aliases.end();)
|
||||
{
|
||||
const ASTIdentifier * parent = identifier;
|
||||
identifier = it->second->as<ASTIdentifier>();
|
||||
@ -156,12 +156,12 @@ const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifie
|
||||
break; /// not a column alias
|
||||
if (identifier == parent)
|
||||
break; /// alias to itself with the same name: 'a as a'
|
||||
if (identifier->compound())
|
||||
if (identifier->supposedToBeCompound())
|
||||
break; /// not an alias. Break to prevent cycle through short names: 'a as b, t1.b as a'
|
||||
|
||||
it = aliases.find(identifier->name);
|
||||
it = aliases.find(identifier->name());
|
||||
if (!max_attempts--)
|
||||
throw Exception("Cannot unroll aliases for '" + identifier->name + "'", ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception("Cannot unroll aliases for '" + identifier->name() + "'", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
return identifier;
|
||||
@ -186,7 +186,7 @@ size_t CollectJoinOnKeysMatcher::getTableForIdentifiers(std::vector<const ASTIde
|
||||
|
||||
if (!membership)
|
||||
{
|
||||
const String & name = identifier->name;
|
||||
const String & name = identifier->name();
|
||||
bool in_left_table = data.left_table.hasColumn(name);
|
||||
bool in_right_table = data.right_table.hasColumn(name);
|
||||
|
||||
|
||||
@ -335,7 +335,6 @@ struct ContextShared
|
||||
std::optional<BackgroundProcessingPool> background_move_pool; /// The thread pool for the background moves performed by the tables.
|
||||
std::optional<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables)
|
||||
std::optional<BackgroundSchedulePool> distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends)
|
||||
std::optional<BackgroundSchedulePool> message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used in kafka streaming)
|
||||
MultiVersion<Macros> macros; /// Substitutions extracted from config.
|
||||
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
|
||||
/// Rules for selecting the compression settings, depending on the size of the part.
|
||||
@ -438,7 +437,6 @@ struct ContextShared
|
||||
schedule_pool.reset();
|
||||
distributed_schedule_pool.reset();
|
||||
ddl_worker.reset();
|
||||
message_broker_schedule_pool.reset();
|
||||
|
||||
/// Stop trace collector if any
|
||||
trace_collector.reset();
|
||||
@ -1441,17 +1439,6 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool()
|
||||
return *shared->distributed_schedule_pool;
|
||||
}
|
||||
|
||||
BackgroundSchedulePool & Context::getMessageBrokerSchedulePool()
|
||||
{
|
||||
auto lock = getLock();
|
||||
if (!shared->message_broker_schedule_pool)
|
||||
shared->message_broker_schedule_pool.emplace(
|
||||
settings.background_message_broker_schedule_pool_size,
|
||||
CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask,
|
||||
"BgMBSchPool");
|
||||
return *shared->message_broker_schedule_pool;
|
||||
}
|
||||
|
||||
void Context::setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker)
|
||||
{
|
||||
auto lock = getLock();
|
||||
@ -1950,7 +1937,7 @@ void Context::checkCanBeDropped(const String & database, const String & table, c
|
||||
<< (force_file_exists ? "exists but not writeable (could not be removed)" : "doesn't exist") << "\n";
|
||||
|
||||
ostr << "How to fix this:\n"
|
||||
<< "1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config and restart ClickHouse\n"
|
||||
<< "1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config\n"
|
||||
<< "2. Either create forcing file " << force_file.path() << " and make sure that ClickHouse has write permission for it.\n"
|
||||
<< "Example:\nsudo touch '" << force_file.path() << "' && sudo chmod 666 '" << force_file.path() << "'";
|
||||
|
||||
|
||||
@ -511,7 +511,6 @@ public:
|
||||
BackgroundProcessingPool & getBackgroundPool();
|
||||
BackgroundProcessingPool & getBackgroundMovePool();
|
||||
BackgroundSchedulePool & getSchedulePool();
|
||||
BackgroundSchedulePool & getMessageBrokerSchedulePool();
|
||||
BackgroundSchedulePool & getDistributedSchedulePool();
|
||||
|
||||
void setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker);
|
||||
|
||||
@ -532,7 +532,7 @@ std::unique_ptr<DDLGuard> DatabaseCatalog::getDDLGuard(const String & database,
|
||||
std::unique_lock lock(ddl_guards_mutex);
|
||||
auto db_guard_iter = ddl_guards.try_emplace(database).first;
|
||||
DatabaseGuard & db_guard = db_guard_iter->second;
|
||||
return std::make_unique<DDLGuard>(db_guard.first, db_guard.second, std::move(lock), table);
|
||||
return std::make_unique<DDLGuard>(db_guard.first, db_guard.second, std::move(lock), table, database);
|
||||
}
|
||||
|
||||
std::unique_lock<std::shared_mutex> DatabaseCatalog::getExclusiveDDLGuardForDatabase(const String & database)
|
||||
@ -834,7 +834,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
|
||||
}
|
||||
|
||||
|
||||
DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem)
|
||||
DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem, const String & database_name)
|
||||
: map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_))
|
||||
{
|
||||
it = map.emplace(elem, Entry{std::make_unique<std::mutex>(), 0}).first;
|
||||
@ -843,14 +843,19 @@ DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<s
|
||||
table_lock = std::unique_lock(*it->second.mutex);
|
||||
bool is_database = elem.empty();
|
||||
if (!is_database)
|
||||
db_mutex.lock_shared();
|
||||
{
|
||||
|
||||
bool locked_database_for_read = db_mutex.try_lock_shared();
|
||||
if (!locked_database_for_read)
|
||||
{
|
||||
removeTableLock();
|
||||
throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} is currently dropped or renamed", database_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DDLGuard::~DDLGuard()
|
||||
void DDLGuard::removeTableLock()
|
||||
{
|
||||
bool is_database = it->first.empty();
|
||||
if (!is_database)
|
||||
db_mutex.unlock_shared();
|
||||
guards_lock.lock();
|
||||
--it->second.counter;
|
||||
if (!it->second.counter)
|
||||
@ -860,4 +865,12 @@ DDLGuard::~DDLGuard()
|
||||
}
|
||||
}
|
||||
|
||||
DDLGuard::~DDLGuard()
|
||||
{
|
||||
bool is_database = it->first.empty();
|
||||
if (!is_database)
|
||||
db_mutex.unlock_shared();
|
||||
removeTableLock();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -51,7 +51,7 @@ public:
|
||||
/// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion.
|
||||
using Map = std::map<String, Entry>;
|
||||
|
||||
DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem);
|
||||
DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem, const String & database_name);
|
||||
~DDLGuard();
|
||||
|
||||
private:
|
||||
@ -60,6 +60,8 @@ private:
|
||||
Map::iterator it;
|
||||
std::unique_lock<std::mutex> guards_lock;
|
||||
std::unique_lock<std::mutex> table_lock;
|
||||
|
||||
void removeTableLock();
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -316,6 +316,7 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
|
||||
{
|
||||
auto & result = sample_block.getByName(result_name);
|
||||
result.type = result_type;
|
||||
result.name = result_name;
|
||||
result.column = source.column;
|
||||
}
|
||||
else
|
||||
@ -1560,6 +1561,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction(
|
||||
ColumnWithTypeAndName argument;
|
||||
argument.column = child.column;
|
||||
argument.type = child.result_type;
|
||||
argument.name = child.result_name;
|
||||
|
||||
if (!argument.column || !isColumnConst(*argument.column))
|
||||
all_const = false;
|
||||
|
||||
@ -41,7 +41,7 @@ void ExpressionInfoMatcher::visit(const ASTIdentifier & identifier, const ASTPtr
|
||||
const auto & table = data.tables[index];
|
||||
|
||||
// TODO: make sure no collision ever happens
|
||||
if (table.hasColumn(identifier.name))
|
||||
if (table.hasColumn(identifier.name()))
|
||||
{
|
||||
data.unique_reference_tables_pos.emplace(index);
|
||||
break;
|
||||
|
||||
@ -51,7 +51,7 @@ std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std
|
||||
if ((best_match != ColumnMatch::NoMatch) && same_match)
|
||||
{
|
||||
if (!allow_ambiguous)
|
||||
throw Exception("Ambiguous column '" + identifier.name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
throw Exception("Ambiguous column '" + identifier.name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
best_match = ColumnMatch::Ambiguous;
|
||||
return {};
|
||||
}
|
||||
@ -66,7 +66,7 @@ std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std
|
||||
std::optional<String> IdentifierSemantic::getColumnName(const ASTIdentifier & node)
|
||||
{
|
||||
if (!node.semantic->special)
|
||||
return node.name;
|
||||
return node.name();
|
||||
return {};
|
||||
}
|
||||
|
||||
@ -75,14 +75,14 @@ std::optional<String> IdentifierSemantic::getColumnName(const ASTPtr & ast)
|
||||
if (ast)
|
||||
if (const auto * id = ast->as<ASTIdentifier>())
|
||||
if (!id->semantic->special)
|
||||
return id->name;
|
||||
return id->name();
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<String> IdentifierSemantic::getTableName(const ASTIdentifier & node)
|
||||
{
|
||||
if (node.semantic->special)
|
||||
return node.name;
|
||||
return node.name();
|
||||
return {};
|
||||
}
|
||||
|
||||
@ -91,7 +91,7 @@ std::optional<String> IdentifierSemantic::getTableName(const ASTPtr & ast)
|
||||
if (ast)
|
||||
if (const auto * id = ast->as<ASTIdentifier>())
|
||||
if (id->semantic->special)
|
||||
return id->name;
|
||||
return id->name();
|
||||
return {};
|
||||
}
|
||||
|
||||
@ -151,7 +151,7 @@ StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & iden
|
||||
|
||||
if (identifier.name_parts.size() == 2)
|
||||
return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid };
|
||||
return { "", identifier.name, identifier.uuid };
|
||||
return { "", identifier.name_parts[0], identifier.uuid };
|
||||
}
|
||||
|
||||
std::optional<String> IdentifierSemantic::extractNestedName(const ASTIdentifier & identifier, const String & table_name)
|
||||
@ -232,16 +232,8 @@ void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const Da
|
||||
if (!to_strip)
|
||||
return;
|
||||
|
||||
std::vector<String> stripped(identifier.name_parts.begin() + to_strip, identifier.name_parts.end());
|
||||
|
||||
DB::String new_name;
|
||||
for (const auto & part : stripped)
|
||||
{
|
||||
if (!new_name.empty())
|
||||
new_name += '.';
|
||||
new_name += part;
|
||||
}
|
||||
identifier.name.swap(new_name);
|
||||
identifier.name_parts = std::vector<String>(identifier.name_parts.begin() + to_strip, identifier.name_parts.end());
|
||||
identifier.resetFullName();
|
||||
}
|
||||
|
||||
void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table)
|
||||
@ -249,10 +241,11 @@ void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const Dat
|
||||
String prefix = db_and_table.getQualifiedNamePrefix();
|
||||
if (!prefix.empty())
|
||||
{
|
||||
String short_name = identifier.shortName();
|
||||
identifier.name = prefix + short_name;
|
||||
prefix.resize(prefix.size() - 1); /// crop dot
|
||||
identifier.name_parts = {prefix, short_name};
|
||||
identifier.name_parts = {prefix, identifier.shortName()};
|
||||
identifier.resetFullName();
|
||||
identifier.semantic->table = prefix;
|
||||
identifier.semantic->legacy_compound = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -14,6 +14,8 @@ struct IdentifierSemanticImpl
|
||||
bool can_be_alias = true; /// if it's a cropped name it could not be an alias
|
||||
bool covered = false; /// real (compound) name is hidden by an alias (short name)
|
||||
std::optional<size_t> membership; /// table position in join
|
||||
String table = {}; /// store table name for columns just to support legacy logic.
|
||||
bool legacy_compound = false; /// true if identifier supposed to be comply for legacy |compound()| behavior
|
||||
};
|
||||
|
||||
/// Static class to manipulate IdentifierSemanticImpl via ASTIdentifier
|
||||
|
||||
@ -29,7 +29,15 @@ public:
|
||||
if constexpr (!_top_to_bottom)
|
||||
visitChildren(ast);
|
||||
|
||||
try
|
||||
{
|
||||
Matcher::visit(ast, data);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage("While processing {}", ast->formatForErrorMessage());
|
||||
throw;
|
||||
}
|
||||
|
||||
if constexpr (_top_to_bottom)
|
||||
visitChildren(ast);
|
||||
|
||||
@ -47,40 +47,37 @@ BlockIO InterpreterDropQuery::execute()
|
||||
if (!drop.table.empty())
|
||||
{
|
||||
if (!drop.is_dictionary)
|
||||
return executeToTable({drop.database, drop.table, drop.uuid}, drop);
|
||||
return executeToTable(drop);
|
||||
else
|
||||
return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock);
|
||||
}
|
||||
else if (!drop.database.empty())
|
||||
return executeToDatabase(drop.database, drop.kind, drop.if_exists);
|
||||
return executeToDatabase(drop.database, drop.kind, drop.if_exists, drop.no_delay);
|
||||
else
|
||||
throw Exception("Nothing to drop, both names are empty", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
||||
BlockIO InterpreterDropQuery::executeToTable(
|
||||
const StorageID & table_id_,
|
||||
const ASTDropQuery & query)
|
||||
BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query)
|
||||
{
|
||||
if (query.temporary || table_id_.database_name.empty())
|
||||
/// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard
|
||||
auto table_id = StorageID(query);
|
||||
if (query.temporary || table_id.database_name.empty())
|
||||
{
|
||||
if (context.tryResolveStorageID(table_id_, Context::ResolveExternal))
|
||||
return executeToTemporaryTable(table_id_.getTableName(), query.kind);
|
||||
if (context.tryResolveStorageID(table_id, Context::ResolveExternal))
|
||||
return executeToTemporaryTable(table_id.getTableName(), query.kind);
|
||||
else
|
||||
table_id.database_name = context.getCurrentDatabase();
|
||||
}
|
||||
|
||||
if (query.temporary)
|
||||
{
|
||||
if (query.if_exists)
|
||||
return {};
|
||||
throw Exception("Temporary table " + backQuoteIfNeed(table_id_.table_name) + " doesn't exist",
|
||||
throw Exception("Temporary table " + backQuoteIfNeed(table_id.table_name) + " doesn't exist",
|
||||
ErrorCodes::UNKNOWN_TABLE);
|
||||
}
|
||||
|
||||
auto table_id = query.if_exists ? context.tryResolveStorageID(table_id_, Context::ResolveOrdinary)
|
||||
: context.resolveStorageID(table_id_, Context::ResolveOrdinary);
|
||||
if (!table_id)
|
||||
return {};
|
||||
|
||||
auto ddl_guard = (!query.no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name) : nullptr);
|
||||
|
||||
/// If table was already dropped by anyone, an exception will be thrown
|
||||
@ -92,6 +89,9 @@ BlockIO InterpreterDropQuery::executeToTable(
|
||||
if (query_ptr->as<ASTDropQuery &>().is_view && !table->isView())
|
||||
throw Exception("Table " + table_id.getNameForLogs() + " is not a View", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
/// Now get UUID, so we can wait for table data to be finally dropped
|
||||
table_id.uuid = database->tryGetTableUUID(table_id.table_name);
|
||||
|
||||
if (query.kind == ASTDropQuery::Kind::Detach)
|
||||
{
|
||||
context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id);
|
||||
@ -223,7 +223,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name,
|
||||
}
|
||||
|
||||
|
||||
BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists)
|
||||
BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay)
|
||||
{
|
||||
auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
|
||||
|
||||
@ -251,11 +251,16 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS
|
||||
|
||||
ASTDropQuery query;
|
||||
query.kind = kind;
|
||||
query.if_exists = true;
|
||||
query.database = database_name;
|
||||
query.no_delay = no_delay;
|
||||
|
||||
for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
|
||||
{
|
||||
/// Reset reference counter of the StoragePtr to allow synchronous drop.
|
||||
iterator->reset();
|
||||
query.table = iterator->name();
|
||||
executeToTable({query.database, query.table}, query);
|
||||
executeToTable(query);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,9 +29,9 @@ private:
|
||||
ASTPtr query_ptr;
|
||||
Context & context;
|
||||
|
||||
BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists);
|
||||
BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay);
|
||||
|
||||
BlockIO executeToTable(const StorageID & table_id, const ASTDropQuery & query);
|
||||
BlockIO executeToTable(const ASTDropQuery & query);
|
||||
|
||||
BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock);
|
||||
|
||||
|
||||
@ -117,7 +117,7 @@ private:
|
||||
throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
|
||||
ASTIdentifier & identifier = child->children[0]->as<ASTIdentifier &>();
|
||||
|
||||
data.addTableColumns(identifier.name);
|
||||
data.addTableColumns(identifier.name());
|
||||
}
|
||||
else
|
||||
data.new_select_expression_list->children.push_back(child);
|
||||
@ -228,7 +228,7 @@ struct CollectColumnIdentifiersMatcher
|
||||
void addIdentirier(const ASTIdentifier & ident)
|
||||
{
|
||||
for (const auto & aliases : ignored)
|
||||
if (aliases.count(ident.name))
|
||||
if (aliases.count(ident.name()))
|
||||
return;
|
||||
identifiers.push_back(const_cast<ASTIdentifier *>(&ident));
|
||||
}
|
||||
@ -293,7 +293,7 @@ struct CheckAliasDependencyVisitorData
|
||||
|
||||
void visit(ASTIdentifier & ident, ASTPtr &)
|
||||
{
|
||||
if (!dependency && aliases.count(ident.name))
|
||||
if (!dependency && aliases.count(ident.name()))
|
||||
dependency = &ident;
|
||||
}
|
||||
};
|
||||
@ -467,7 +467,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
|
||||
|
||||
for (ASTIdentifier * ident : identifiers)
|
||||
{
|
||||
bool got_alias = aliases.count(ident->name);
|
||||
bool got_alias = aliases.count(ident->name());
|
||||
bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias
|
||||
|
||||
if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous))
|
||||
@ -475,12 +475,12 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
|
||||
if (!ident->isShort())
|
||||
{
|
||||
if (got_alias)
|
||||
throw Exception("Alias clashes with qualified column '" + ident->name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
throw Exception("Alias clashes with qualified column '" + ident->name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
|
||||
String short_name = ident->shortName();
|
||||
String original_long_name;
|
||||
if (public_identifiers.count(ident))
|
||||
original_long_name = ident->name;
|
||||
original_long_name = ident->name();
|
||||
|
||||
size_t count = countTablesWithColumn(tables, short_name);
|
||||
|
||||
@ -488,7 +488,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
|
||||
{
|
||||
const auto & table = tables[*table_pos];
|
||||
IdentifierSemantic::setColumnLongName(*ident, table.table); /// table.column -> table_alias.column
|
||||
auto & unique_long_name = ident->name;
|
||||
const auto & unique_long_name = ident->name();
|
||||
|
||||
/// For tables moved into subselects we need unique short names for clashed names
|
||||
if (*table_pos != last_table_pos)
|
||||
@ -512,7 +512,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
|
||||
needed_columns[*table_pos].no_clashes.emplace(ident->shortName());
|
||||
}
|
||||
else if (!got_alias)
|
||||
throw Exception("Unknown column name '" + ident->name + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
throw Exception("Unknown column name '" + ident->name() + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
}
|
||||
|
||||
return needed_columns;
|
||||
@ -613,12 +613,12 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast
|
||||
{
|
||||
for (auto * ident : on_identifiers)
|
||||
{
|
||||
auto it = data.aliases.find(ident->name);
|
||||
if (!on_aliases.count(ident->name) && it != data.aliases.end())
|
||||
auto it = data.aliases.find(ident->name());
|
||||
if (!on_aliases.count(ident->name()) && it != data.aliases.end())
|
||||
{
|
||||
auto alias_expression = it->second;
|
||||
alias_pushdown[table_pos].push_back(alias_expression);
|
||||
on_aliases[ident->name] = alias_expression;
|
||||
on_aliases[ident->name()] = alias_expression;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -638,14 +638,14 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast
|
||||
CheckAliasDependencyVisitor(check).visit(expr.second);
|
||||
if (check.dependency)
|
||||
throw Exception("Cannot rewrite JOINs. Alias '" + expr.first +
|
||||
"' used in ON section depends on another alias '" + check.dependency->name + "'",
|
||||
"' used in ON section depends on another alias '" + check.dependency->name() + "'",
|
||||
ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// Check same name in aliases, USING and ON sections. Cannot push down alias to ON through USING cause of name masquerading.
|
||||
for (auto * ident : using_identifiers)
|
||||
if (on_aliases.count(ident->name))
|
||||
throw Exception("Cannot rewrite JOINs. Alias '" + ident->name + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED);
|
||||
if (on_aliases.count(ident->name()))
|
||||
throw Exception("Cannot rewrite JOINs. Alias '" + ident->name() + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED);
|
||||
using_identifiers.clear();
|
||||
|
||||
/// Replace pushdowned expressions with aliases names in original expression lists.
|
||||
|
||||
@ -49,7 +49,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query)
|
||||
if (table_expr.database_and_table_name)
|
||||
{
|
||||
const auto & table_id = table_expr.database_and_table_name->as<ASTIdentifier &>();
|
||||
String expr = "(select * from " + table_id.name + ") as " + table_id.shortName();
|
||||
String expr = "(select * from " + table_id.name() + ") as " + table_id.shortName();
|
||||
|
||||
// FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b",
|
||||
// we can't replace aliased tables.
|
||||
@ -99,7 +99,7 @@ private:
|
||||
match == IdentifierSemantic::ColumnMatch::DbAndTable)
|
||||
{
|
||||
if (rewritten)
|
||||
throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name + "'",
|
||||
throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name() + "'",
|
||||
ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
/// Table has an alias. So we set a new name qualified by table alias.
|
||||
IdentifierSemantic::setColumnLongName(identifier, table);
|
||||
@ -114,10 +114,10 @@ private:
|
||||
bool rewritten = false;
|
||||
for (const auto & table : data)
|
||||
{
|
||||
if (identifier.name == table.table)
|
||||
if (identifier.name() == table.table)
|
||||
{
|
||||
if (rewritten)
|
||||
throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name + "'",
|
||||
throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name() + "'",
|
||||
ErrorCodes::AMBIGUOUS_COLUMN_NAME);
|
||||
identifier.setShortName(table.alias);
|
||||
rewritten = true;
|
||||
|
||||
@ -73,8 +73,8 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
|
||||
return;
|
||||
|
||||
/// If it is an alias, but not a parent alias (for constructs like "SELECT column + 1 AS column").
|
||||
auto it_alias = data.aliases.find(node.name);
|
||||
if (it_alias != data.aliases.end() && current_alias != node.name)
|
||||
auto it_alias = data.aliases.find(node.name());
|
||||
if (it_alias != data.aliases.end() && current_alias != node.name())
|
||||
{
|
||||
if (!IdentifierSemantic::canBeAlias(node))
|
||||
return;
|
||||
@ -89,7 +89,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
|
||||
String node_alias = ast->tryGetAlias();
|
||||
|
||||
if (current_asts.count(alias_node.get()) /// We have loop of multiple aliases
|
||||
|| (node.name == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop
|
||||
|| (node.name() == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop
|
||||
throw Exception("Cyclic aliases", ErrorCodes::CYCLIC_ALIASES);
|
||||
|
||||
/// Let's replace it with the corresponding tree node.
|
||||
@ -97,7 +97,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
|
||||
{
|
||||
/// Avoid infinite recursion here
|
||||
auto opt_name = IdentifierSemantic::getColumnName(alias_node);
|
||||
bool is_cycle = opt_name && *opt_name == node.name;
|
||||
bool is_cycle = opt_name && *opt_name == node.name();
|
||||
|
||||
if (!is_cycle)
|
||||
{
|
||||
|
||||
@ -3,10 +3,13 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void RenameColumnData::visit(ASTIdentifier & identifier, ASTPtr &) const
|
||||
{
|
||||
// TODO(ilezhankin): make proper rename
|
||||
std::optional<String> identifier_column_name = IdentifierSemantic::getColumnName(identifier);
|
||||
if (identifier_column_name && identifier_column_name == column_name)
|
||||
identifier.name = rename_to;
|
||||
identifier.setShortName(rename_to);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -27,7 +27,7 @@ void RequiredSourceColumnsData::addColumnIdentifier(const ASTIdentifier & node)
|
||||
|
||||
/// There should be no complex cases after query normalization. Names to aliases: one-to-many.
|
||||
String alias = node.tryGetAlias();
|
||||
required_names[node.name].addInclusion(alias);
|
||||
required_names[node.name()].addInclusion(alias);
|
||||
}
|
||||
|
||||
bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast)
|
||||
@ -42,7 +42,7 @@ bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast)
|
||||
|
||||
void RequiredSourceColumnsData::addArrayJoinIdentifier(const ASTIdentifier & node)
|
||||
{
|
||||
array_join_columns.insert(node.name);
|
||||
array_join_columns.insert(node.name());
|
||||
}
|
||||
|
||||
size_t RequiredSourceColumnsData::nameInclusion(const String & name) const
|
||||
|
||||
@ -34,7 +34,7 @@ std::vector<String> RequiredSourceColumnsMatcher::extractNamesFromLambda(const A
|
||||
if (!identifier)
|
||||
throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH);
|
||||
|
||||
names.push_back(identifier->name);
|
||||
names.push_back(identifier->name());
|
||||
}
|
||||
|
||||
return names;
|
||||
@ -132,10 +132,11 @@ void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const AS
|
||||
|
||||
void RequiredSourceColumnsMatcher::visit(const ASTIdentifier & node, const ASTPtr &, Data & data)
|
||||
{
|
||||
if (node.name.empty())
|
||||
// FIXME(ilezhankin): shouldn't ever encounter
|
||||
if (node.name().empty())
|
||||
throw Exception("Expected not empty name", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (!data.private_aliases.count(node.name))
|
||||
if (!data.private_aliases.count(node.name()))
|
||||
data.addColumnIdentifier(node);
|
||||
}
|
||||
|
||||
|
||||
@ -300,8 +300,8 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
|
||||
performance_counters.setParent(&ProfileEvents::global_counters);
|
||||
memory_tracker.reset();
|
||||
|
||||
/// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below.
|
||||
memory_tracker.setParent(nullptr);
|
||||
/// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below (will reset to its parent).
|
||||
memory_tracker.setParent(thread_group->memory_tracker.getParent());
|
||||
|
||||
query_id.clear();
|
||||
query_context = nullptr;
|
||||
|
||||
@ -104,7 +104,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &,
|
||||
if (data.unknownColumn(table_pos, identifier))
|
||||
{
|
||||
String table_name = data.tables[table_pos].table.getQualifiedNamePrefix(false);
|
||||
throw Exception("There's no column '" + identifier.name + "' in table '" + table_name + "'",
|
||||
throw Exception("There's no column '" + identifier.name() + "' in table '" + table_name + "'",
|
||||
ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
}
|
||||
|
||||
@ -175,9 +175,12 @@ void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr
|
||||
|
||||
static void addIdentifier(ASTs & nodes, const DatabaseAndTableWithAlias & table, const String & column_name)
|
||||
{
|
||||
std::vector<String> parts = {column_name};
|
||||
|
||||
String table_name = table.getQualifiedNamePrefix(false);
|
||||
auto identifier = std::make_shared<ASTIdentifier>(std::vector<String>{table_name, column_name});
|
||||
nodes.emplace_back(identifier);
|
||||
if (!table_name.empty()) parts.insert(parts.begin(), table_name);
|
||||
|
||||
nodes.emplace_back(std::make_shared<ASTIdentifier>(std::move(parts)));
|
||||
}
|
||||
|
||||
/// Replace *, alias.*, database.table.* with a list of columns.
|
||||
@ -354,7 +357,7 @@ void RestoreQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, D
|
||||
{
|
||||
if (IdentifierSemantic::getMembership(identifier))
|
||||
{
|
||||
identifier.restoreCompoundName();
|
||||
identifier.restoreTable(); // TODO(ilezhankin): should restore qualified name here - why exactly here?
|
||||
if (data.rename)
|
||||
data.changeTable(identifier);
|
||||
}
|
||||
|
||||
@ -72,7 +72,7 @@ ASTPtr evaluateConstantExpressionAsLiteral(const ASTPtr & node, const Context &
|
||||
ASTPtr evaluateConstantExpressionOrIdentifierAsLiteral(const ASTPtr & node, const Context & context)
|
||||
{
|
||||
if (const auto * id = node->as<ASTIdentifier>())
|
||||
return std::make_shared<ASTLiteral>(id->name);
|
||||
return std::make_shared<ASTLiteral>(id->name());
|
||||
|
||||
return evaluateConstantExpressionAsLiteral(node, context);
|
||||
}
|
||||
@ -113,7 +113,7 @@ namespace
|
||||
const auto & name = name_and_type.name;
|
||||
const auto & type = name_and_type.type;
|
||||
|
||||
if (name == identifier->name)
|
||||
if (name == identifier->name())
|
||||
{
|
||||
ColumnWithTypeAndName column;
|
||||
Field converted = convertFieldToType(value, *type);
|
||||
|
||||
@ -19,7 +19,7 @@ namespace ErrorCodes
|
||||
std::string getClusterName(const IAST & node)
|
||||
{
|
||||
if (const auto * ast_id = node.as<ASTIdentifier>())
|
||||
return ast_id->name;
|
||||
return ast_id->name();
|
||||
|
||||
if (const auto * ast_lit = node.as<ASTLiteral>())
|
||||
return ast_lit->value.safeGet<String>();
|
||||
|
||||
@ -128,7 +128,7 @@ int main(int argc, char ** argv)
|
||||
std::cerr << "sum_counts: " << sum_counts << ", elems: " << elems << std::endl;
|
||||
|
||||
if (sum_counts != n)
|
||||
std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
|
||||
std::cerr << "Error!" << std::endl;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
@ -71,7 +71,7 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
|
||||
{
|
||||
for (const auto & except_child : children)
|
||||
{
|
||||
if (except_child->as<const ASTIdentifier &>().name == id->shortName())
|
||||
if (except_child->as<const ASTIdentifier &>().name() == id->shortName())
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
#include <IO/WriteBufferFromOStream.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/IdentifierSemantic.h>
|
||||
#include <Interpreters/StorageID.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -16,6 +16,27 @@ namespace ErrorCodes
|
||||
extern const int SYNTAX_ERROR;
|
||||
}
|
||||
|
||||
ASTIdentifier::ASTIdentifier(const String & short_name)
|
||||
: full_name(short_name), name_parts{short_name}, semantic(std::make_shared<IdentifierSemanticImpl>())
|
||||
{
|
||||
assert(!full_name.empty());
|
||||
}
|
||||
|
||||
ASTIdentifier::ASTIdentifier(std::vector<String> && name_parts_, bool special)
|
||||
: name_parts(name_parts_), semantic(std::make_shared<IdentifierSemanticImpl>())
|
||||
{
|
||||
assert(!name_parts.empty());
|
||||
for (const auto & part [[maybe_unused]] : name_parts)
|
||||
assert(!part.empty());
|
||||
|
||||
semantic->special = special;
|
||||
semantic->legacy_compound = true;
|
||||
|
||||
if (!special && name_parts.size() >= 2)
|
||||
semantic->table = name_parts.end()[-2];
|
||||
|
||||
resetFullName();
|
||||
}
|
||||
|
||||
ASTPtr ASTIdentifier::clone() const
|
||||
{
|
||||
@ -24,51 +45,29 @@ ASTPtr ASTIdentifier::clone() const
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::shared_ptr<ASTIdentifier> ASTIdentifier::createSpecial(const String & name, std::vector<String> && name_parts)
|
||||
bool ASTIdentifier::supposedToBeCompound() const
|
||||
{
|
||||
auto ret = std::make_shared<ASTIdentifier>(name, std::move(name_parts));
|
||||
ret->semantic->special = true;
|
||||
return ret;
|
||||
return semantic->legacy_compound;
|
||||
}
|
||||
|
||||
ASTIdentifier::ASTIdentifier(const String & name_, std::vector<String> && name_parts_)
|
||||
: name(name_)
|
||||
, name_parts(name_parts_)
|
||||
, semantic(std::make_shared<IdentifierSemanticImpl>())
|
||||
{
|
||||
if (!name_parts.empty() && name_parts[0].empty())
|
||||
name_parts.erase(name_parts.begin());
|
||||
|
||||
if (name.empty())
|
||||
{
|
||||
if (name_parts.size() == 2)
|
||||
name = name_parts[0] + '.' + name_parts[1];
|
||||
else if (name_parts.size() == 1)
|
||||
name = name_parts[0];
|
||||
}
|
||||
}
|
||||
|
||||
ASTIdentifier::ASTIdentifier(std::vector<String> && name_parts_)
|
||||
: ASTIdentifier("", std::move(name_parts_))
|
||||
{}
|
||||
|
||||
void ASTIdentifier::setShortName(const String & new_name)
|
||||
{
|
||||
name = new_name;
|
||||
name_parts.clear();
|
||||
assert(!new_name.empty());
|
||||
|
||||
full_name = new_name;
|
||||
name_parts = {new_name};
|
||||
|
||||
bool special = semantic->special;
|
||||
*semantic = IdentifierSemanticImpl();
|
||||
semantic->special = special;
|
||||
}
|
||||
|
||||
void ASTIdentifier::restoreCompoundName()
|
||||
const String & ASTIdentifier::name() const
|
||||
{
|
||||
if (name_parts.empty())
|
||||
return;
|
||||
name = name_parts[0];
|
||||
for (size_t i = 1; i < name_parts.size(); ++i)
|
||||
name += '.' + name_parts[i];
|
||||
assert(!name_parts.empty());
|
||||
assert(!full_name.empty());
|
||||
|
||||
return full_name;
|
||||
}
|
||||
|
||||
void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const
|
||||
@ -93,20 +92,29 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form
|
||||
}
|
||||
else
|
||||
{
|
||||
format_element(name);
|
||||
format_element(shortName());
|
||||
}
|
||||
}
|
||||
|
||||
void ASTIdentifier::appendColumnNameImpl(WriteBuffer & ostr) const
|
||||
{
|
||||
writeString(name, ostr);
|
||||
writeString(name(), ostr);
|
||||
}
|
||||
|
||||
void ASTIdentifier::restoreTable()
|
||||
{
|
||||
if (!compound())
|
||||
{
|
||||
name_parts.insert(name_parts.begin(), semantic->table);
|
||||
resetFullName();
|
||||
}
|
||||
}
|
||||
|
||||
void ASTIdentifier::resetTable(const String & database_name, const String & table_name)
|
||||
{
|
||||
auto ast = createTableIdentifier(database_name, table_name);
|
||||
auto & ident = ast->as<ASTIdentifier &>();
|
||||
name.swap(ident.name);
|
||||
full_name.swap(ident.full_name);
|
||||
name_parts.swap(ident.name_parts);
|
||||
uuid = ident.uuid;
|
||||
}
|
||||
@ -117,6 +125,13 @@ void ASTIdentifier::updateTreeHashImpl(SipHash & hash_state) const
|
||||
IAST::updateTreeHashImpl(hash_state);
|
||||
}
|
||||
|
||||
void ASTIdentifier::resetFullName()
|
||||
{
|
||||
full_name = name_parts[0];
|
||||
for (size_t i = 1; i < name_parts.size(); ++i)
|
||||
full_name += '.' + name_parts[i];
|
||||
}
|
||||
|
||||
ASTPtr createTableIdentifier(const String & database_name, const String & table_name)
|
||||
{
|
||||
assert(database_name != "_temporary_and_external_tables");
|
||||
@ -127,9 +142,9 @@ ASTPtr createTableIdentifier(const StorageID & table_id)
|
||||
{
|
||||
std::shared_ptr<ASTIdentifier> res;
|
||||
if (table_id.database_name.empty())
|
||||
res = ASTIdentifier::createSpecial(table_id.table_name);
|
||||
res = std::make_shared<ASTIdentifier>(std::vector<String>{table_id.table_name}, true);
|
||||
else
|
||||
res = ASTIdentifier::createSpecial(table_id.database_name + "." + table_id.table_name, {table_id.database_name, table_id.table_name});
|
||||
res = std::make_shared<ASTIdentifier>(std::vector<String>{table_id.database_name, table_id.table_name}, true);
|
||||
res->uuid = table_id.uuid;
|
||||
return res;
|
||||
}
|
||||
@ -156,7 +171,7 @@ bool tryGetIdentifierNameInto(const IAST * ast, String & name)
|
||||
{
|
||||
if (const auto * node = ast->as<ASTIdentifier>())
|
||||
{
|
||||
name = node->name;
|
||||
name = node->name();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -180,7 +195,7 @@ StorageID getTableIdentifier(const ASTPtr & ast)
|
||||
|
||||
if (identifier.name_parts.size() == 2)
|
||||
return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid };
|
||||
return { "", identifier.name, identifier.uuid };
|
||||
return { "", identifier.name_parts[0], identifier.uuid };
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -18,59 +18,54 @@ struct StorageID;
|
||||
class ASTIdentifier : public ASTWithAlias
|
||||
{
|
||||
public:
|
||||
/// The composite identifier will have a concatenated name (of the form a.b.c),
|
||||
/// and individual components will be available inside the name_parts.
|
||||
String name;
|
||||
UUID uuid = UUIDHelpers::Nil;
|
||||
|
||||
ASTIdentifier(const String & name_, std::vector<String> && name_parts_ = {});
|
||||
ASTIdentifier(std::vector<String> && name_parts_);
|
||||
explicit ASTIdentifier(const String & short_name);
|
||||
explicit ASTIdentifier(std::vector<String> && name_parts, bool special = false);
|
||||
|
||||
/** Get the text that identifies this element. */
|
||||
String getID(char delim) const override { return "Identifier" + (delim + name); }
|
||||
String getID(char delim) const override { return "Identifier" + (delim + name()); }
|
||||
|
||||
ASTPtr clone() const override;
|
||||
|
||||
void collectIdentifierNames(IdentifierNameSet & set) const override
|
||||
{
|
||||
set.insert(name);
|
||||
}
|
||||
void collectIdentifierNames(IdentifierNameSet & set) const override { set.insert(name()); }
|
||||
|
||||
bool compound() const { return !name_parts.empty(); }
|
||||
bool isShort() const { return name_parts.empty() || name == name_parts.back(); }
|
||||
bool compound() const { return name_parts.size() > 1; }
|
||||
bool isShort() const { return name_parts.size() == 1; }
|
||||
bool supposedToBeCompound() const; // TODO(ilezhankin): get rid of this
|
||||
|
||||
void setShortName(const String & new_name);
|
||||
|
||||
/// Restore name field from name_parts in case it was cropped by analyzer but we need a full form for future (re)analyze.
|
||||
void restoreCompoundName();
|
||||
/// The composite identifier will have a concatenated name (of the form a.b.c),
|
||||
/// and individual components will be available inside the name_parts.
|
||||
const String & shortName() const { return name_parts.back(); }
|
||||
const String & name() const;
|
||||
|
||||
const String & shortName() const
|
||||
{
|
||||
if (!name_parts.empty())
|
||||
return name_parts.back();
|
||||
return name;
|
||||
}
|
||||
void restoreTable(); // TODO(ilezhankin): get rid of this
|
||||
|
||||
void resetTable(const String & database_name, const String & table_name);
|
||||
// FIXME: used only when it's needed to rewrite distributed table name to real remote table name.
|
||||
void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this
|
||||
|
||||
void updateTreeHashImpl(SipHash & hash_state) const override;
|
||||
|
||||
protected:
|
||||
String full_name;
|
||||
std::vector<String> name_parts;
|
||||
|
||||
void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||
void appendColumnNameImpl(WriteBuffer & ostr) const override;
|
||||
|
||||
private:
|
||||
using ASTWithAlias::children; /// ASTIdentifier is child free
|
||||
|
||||
std::vector<String> name_parts;
|
||||
std::shared_ptr<IdentifierSemanticImpl> semantic; /// pimpl
|
||||
|
||||
static std::shared_ptr<ASTIdentifier> createSpecial(const String & name, std::vector<String> && name_parts = {});
|
||||
|
||||
friend struct IdentifierSemantic;
|
||||
friend ASTPtr createTableIdentifier(const StorageID & table_id);
|
||||
friend void setIdentifierSpecial(ASTPtr & ast);
|
||||
friend StorageID getTableIdentifier(const ASTPtr & ast);
|
||||
|
||||
void resetFullName();
|
||||
};
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user