Merge remote-tracking branch 'origin/master' into dev_replace

This commit is contained in:
hexiaoting 2020-10-26 11:27:55 +08:00
commit 333cf8298f
177 changed files with 3101 additions and 21414 deletions

4
.gitmodules vendored
View File

@ -186,3 +186,7 @@
path = contrib/cyrus-sasl
url = https://github.com/cyrusimap/cyrus-sasl
branch = cyrus-sasl-2.1
[submodule "contrib/croaring"]
path = contrib/croaring
url = https://github.com/RoaringBitmap/CRoaring
branch = v0.2.66

View File

@ -0,0 +1,339 @@
/* origin: OpenBSD /usr/src/lib/libm/src/ld80/e_lgammal.c */
/*
* ====================================================
* Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
*
* Developed at SunPro, a Sun Microsystems, Inc. business.
* Permission to use, copy, modify, and distribute this
* software is freely granted, provided that this notice
* is preserved.
* ====================================================
*/
/*
* Copyright (c) 2008 Stephen L. Moshier <steve@moshier.net>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/* lgammal(x)
* Reentrant version of the logarithm of the Gamma function
* with user provide pointer for the sign of Gamma(x).
*
* Method:
* 1. Argument Reduction for 0 < x <= 8
* Since gamma(1+s)=s*gamma(s), for x in [0,8], we may
* reduce x to a number in [1.5,2.5] by
* lgamma(1+s) = log(s) + lgamma(s)
* for example,
* lgamma(7.3) = log(6.3) + lgamma(6.3)
* = log(6.3*5.3) + lgamma(5.3)
* = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3)
* 2. Polynomial approximation of lgamma around its
* minimun ymin=1.461632144968362245 to maintain monotonicity.
* On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use
* Let z = x-ymin;
* lgamma(x) = -1.214862905358496078218 + z^2*poly(z)
* 2. Rational approximation in the primary interval [2,3]
* We use the following approximation:
* s = x-2.0;
* lgamma(x) = 0.5*s + s*P(s)/Q(s)
* Our algorithms are based on the following observation
*
* zeta(2)-1 2 zeta(3)-1 3
* lgamma(2+s) = s*(1-Euler) + --------- * s - --------- * s + ...
* 2 3
*
* where Euler = 0.5771... is the Euler constant, which is very
* close to 0.5.
*
* 3. For x>=8, we have
* lgamma(x)~(x-0.5)log(x)-x+0.5*log(2pi)+1/(12x)-1/(360x**3)+....
* (better formula:
* lgamma(x)~(x-0.5)*(log(x)-1)-.5*(log(2pi)-1) + ...)
* Let z = 1/x, then we approximation
* f(z) = lgamma(x) - (x-0.5)(log(x)-1)
* by
* 3 5 11
* w = w0 + w1*z + w2*z + w3*z + ... + w6*z
*
* 4. For negative x, since (G is gamma function)
* -x*G(-x)*G(x) = pi/sin(pi*x),
* we have
* G(x) = pi/(sin(pi*x)*(-x)*G(-x))
* since G(-x) is positive, sign(G(x)) = sign(sin(pi*x)) for x<0
* Hence, for x<0, signgam = sign(sin(pi*x)) and
* lgamma(x) = log(|Gamma(x)|)
* = log(pi/(|x*sin(pi*x)|)) - lgamma(-x);
* Note: one should avoid compute pi*(-x) directly in the
* computation of sin(pi*(-x)).
*
* 5. Special Cases
* lgamma(2+s) ~ s*(1-Euler) for tiny s
* lgamma(1)=lgamma(2)=0
* lgamma(x) ~ -log(x) for tiny x
* lgamma(0) = lgamma(inf) = inf
* lgamma(-integer) = +-inf
*
*/
#include <stdint.h>
#include <math.h>
#include "libm.h"
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
double lgamma_r(double x, int *sg);
long double lgammal_r(long double x, int *sg)
{
return lgamma_r(x, sg);
}
#elif LDBL_MANT_DIG == 64 && LDBL_MAX_EXP == 16384
static const long double pi = 3.14159265358979323846264L,
/* lgam(1+x) = 0.5 x + x a(x)/b(x)
-0.268402099609375 <= x <= 0
peak relative error 6.6e-22 */
a0 = -6.343246574721079391729402781192128239938E2L,
a1 = 1.856560238672465796768677717168371401378E3L,
a2 = 2.404733102163746263689288466865843408429E3L,
a3 = 8.804188795790383497379532868917517596322E2L,
a4 = 1.135361354097447729740103745999661157426E2L,
a5 = 3.766956539107615557608581581190400021285E0L,
b0 = 8.214973713960928795704317259806842490498E3L,
b1 = 1.026343508841367384879065363925870888012E4L,
b2 = 4.553337477045763320522762343132210919277E3L,
b3 = 8.506975785032585797446253359230031874803E2L,
b4 = 6.042447899703295436820744186992189445813E1L,
/* b5 = 1.000000000000000000000000000000000000000E0 */
tc = 1.4616321449683623412626595423257213284682E0L,
tf = -1.2148629053584961146050602565082954242826E-1, /* double precision */
/* tt = (tail of tf), i.e. tf + tt has extended precision. */
tt = 3.3649914684731379602768989080467587736363E-18L,
/* lgam ( 1.4616321449683623412626595423257213284682E0 ) =
-1.2148629053584960809551455717769158215135617312999903886372437313313530E-1 */
/* lgam (x + tc) = tf + tt + x g(x)/h(x)
-0.230003726999612341262659542325721328468 <= x
<= 0.2699962730003876587373404576742786715318
peak relative error 2.1e-21 */
g0 = 3.645529916721223331888305293534095553827E-18L,
g1 = 5.126654642791082497002594216163574795690E3L,
g2 = 8.828603575854624811911631336122070070327E3L,
g3 = 5.464186426932117031234820886525701595203E3L,
g4 = 1.455427403530884193180776558102868592293E3L,
g5 = 1.541735456969245924860307497029155838446E2L,
g6 = 4.335498275274822298341872707453445815118E0L,
h0 = 1.059584930106085509696730443974495979641E4L,
h1 = 2.147921653490043010629481226937850618860E4L,
h2 = 1.643014770044524804175197151958100656728E4L,
h3 = 5.869021995186925517228323497501767586078E3L,
h4 = 9.764244777714344488787381271643502742293E2L,
h5 = 6.442485441570592541741092969581997002349E1L,
/* h6 = 1.000000000000000000000000000000000000000E0 */
/* lgam (x+1) = -0.5 x + x u(x)/v(x)
-0.100006103515625 <= x <= 0.231639862060546875
peak relative error 1.3e-21 */
u0 = -8.886217500092090678492242071879342025627E1L,
u1 = 6.840109978129177639438792958320783599310E2L,
u2 = 2.042626104514127267855588786511809932433E3L,
u3 = 1.911723903442667422201651063009856064275E3L,
u4 = 7.447065275665887457628865263491667767695E2L,
u5 = 1.132256494121790736268471016493103952637E2L,
u6 = 4.484398885516614191003094714505960972894E0L,
v0 = 1.150830924194461522996462401210374632929E3L,
v1 = 3.399692260848747447377972081399737098610E3L,
v2 = 3.786631705644460255229513563657226008015E3L,
v3 = 1.966450123004478374557778781564114347876E3L,
v4 = 4.741359068914069299837355438370682773122E2L,
v5 = 4.508989649747184050907206782117647852364E1L,
/* v6 = 1.000000000000000000000000000000000000000E0 */
/* lgam (x+2) = .5 x + x s(x)/r(x)
0 <= x <= 1
peak relative error 7.2e-22 */
s0 = 1.454726263410661942989109455292824853344E6L,
s1 = -3.901428390086348447890408306153378922752E6L,
s2 = -6.573568698209374121847873064292963089438E6L,
s3 = -3.319055881485044417245964508099095984643E6L,
s4 = -7.094891568758439227560184618114707107977E5L,
s5 = -6.263426646464505837422314539808112478303E4L,
s6 = -1.684926520999477529949915657519454051529E3L,
r0 = -1.883978160734303518163008696712983134698E7L,
r1 = -2.815206082812062064902202753264922306830E7L,
r2 = -1.600245495251915899081846093343626358398E7L,
r3 = -4.310526301881305003489257052083370058799E6L,
r4 = -5.563807682263923279438235987186184968542E5L,
r5 = -3.027734654434169996032905158145259713083E4L,
r6 = -4.501995652861105629217250715790764371267E2L,
/* r6 = 1.000000000000000000000000000000000000000E0 */
/* lgam(x) = ( x - 0.5 ) * log(x) - x + LS2PI + 1/x w(1/x^2)
x >= 8
Peak relative error 1.51e-21
w0 = LS2PI - 0.5 */
w0 = 4.189385332046727417803e-1L,
w1 = 8.333333333333331447505E-2L,
w2 = -2.777777777750349603440E-3L,
w3 = 7.936507795855070755671E-4L,
w4 = -5.952345851765688514613E-4L,
w5 = 8.412723297322498080632E-4L,
w6 = -1.880801938119376907179E-3L,
w7 = 4.885026142432270781165E-3L;
long double lgammal_r(long double x, int *sg) {
long double t, y, z, nadj, p, p1, p2, q, r, w;
union ldshape u = {x};
uint32_t ix = (u.i.se & 0x7fffU)<<16 | u.i.m>>48;
int sign = u.i.se >> 15;
int i;
*sg = 1;
/* purge off +-inf, NaN, +-0, tiny and negative arguments */
if (ix >= 0x7fff0000)
return x * x;
if (ix < 0x3fc08000) { /* |x|<2**-63, return -log(|x|) */
if (sign) {
*sg = -1;
x = -x;
}
return -logl(x);
}
if (sign) {
x = -x;
t = sin(pi * x);
if (t == 0.0)
return 1.0 / (x-x); /* -integer */
if (t > 0.0)
*sg = -1;
else
t = -t;
nadj = logl(pi / (t * x));
}
/* purge off 1 and 2 (so the sign is ok with downward rounding) */
if ((ix == 0x3fff8000 || ix == 0x40008000) && u.i.m == 0) {
r = 0;
} else if (ix < 0x40008000) { /* x < 2.0 */
if (ix <= 0x3ffee666) { /* 8.99993896484375e-1 */
/* lgamma(x) = lgamma(x+1) - log(x) */
r = -logl(x);
if (ix >= 0x3ffebb4a) { /* 7.31597900390625e-1 */
y = x - 1.0;
i = 0;
} else if (ix >= 0x3ffced33) { /* 2.31639862060546875e-1 */
y = x - (tc - 1.0);
i = 1;
} else { /* x < 0.23 */
y = x;
i = 2;
}
} else {
r = 0.0;
if (ix >= 0x3fffdda6) { /* 1.73162841796875 */
/* [1.7316,2] */
y = x - 2.0;
i = 0;
} else if (ix >= 0x3fff9da6) { /* 1.23162841796875 */
/* [1.23,1.73] */
y = x - tc;
i = 1;
} else {
/* [0.9, 1.23] */
y = x - 1.0;
i = 2;
}
}
switch (i) {
case 0:
p1 = a0 + y * (a1 + y * (a2 + y * (a3 + y * (a4 + y * a5))));
p2 = b0 + y * (b1 + y * (b2 + y * (b3 + y * (b4 + y))));
r += 0.5 * y + y * p1/p2;
break;
case 1:
p1 = g0 + y * (g1 + y * (g2 + y * (g3 + y * (g4 + y * (g5 + y * g6)))));
p2 = h0 + y * (h1 + y * (h2 + y * (h3 + y * (h4 + y * (h5 + y)))));
p = tt + y * p1/p2;
r += (tf + p);
break;
case 2:
p1 = y * (u0 + y * (u1 + y * (u2 + y * (u3 + y * (u4 + y * (u5 + y * u6))))));
p2 = v0 + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * (v5 + y)))));
r += (-0.5 * y + p1 / p2);
}
} else if (ix < 0x40028000) { /* 8.0 */
/* x < 8.0 */
i = (int)x;
y = x - (double)i;
p = y * (s0 + y * (s1 + y * (s2 + y * (s3 + y * (s4 + y * (s5 + y * s6))))));
q = r0 + y * (r1 + y * (r2 + y * (r3 + y * (r4 + y * (r5 + y * (r6 + y))))));
r = 0.5 * y + p / q;
z = 1.0;
/* lgamma(1+s) = log(s) + lgamma(s) */
switch (i) {
case 7:
z *= (y + 6.0); /* FALLTHRU */
case 6:
z *= (y + 5.0); /* FALLTHRU */
case 5:
z *= (y + 4.0); /* FALLTHRU */
case 4:
z *= (y + 3.0); /* FALLTHRU */
case 3:
z *= (y + 2.0); /* FALLTHRU */
r += logl(z);
break;
}
} else if (ix < 0x40418000) { /* 2^66 */
/* 8.0 <= x < 2**66 */
t = logl(x);
z = 1.0 / x;
y = z * z;
w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * (w6 + y * w7))))));
r = (x - 0.5) * (t - 1.0) + w;
} else /* 2**66 <= x <= inf */
r = x * (logl(x) - 1.0);
if (sign)
r = nadj - r;
return r;
}
#elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384
// TODO: broken implementation to make things compile
double lgamma_r(double x, int *sg);
long double lgammal_r(long double x, int *sg)
{
return lgamma_r(x, sg);
}
#endif
int signgam_lgammal;
long double lgammal(long double x)
{
return lgammal_r(x, &signgam_lgammal);
}

View File

@ -20,7 +20,6 @@ add_subdirectory (boost-cmake)
add_subdirectory (cctz-cmake)
add_subdirectory (consistent-hashing-sumbur)
add_subdirectory (consistent-hashing)
add_subdirectory (croaring)
add_subdirectory (FastMemcpy)
add_subdirectory (hyperscan-cmake)
add_subdirectory (jemalloc-cmake)
@ -34,6 +33,7 @@ add_subdirectory (ryu-cmake)
add_subdirectory (unixodbc-cmake)
add_subdirectory (poco-cmake)
add_subdirectory (croaring-cmake)
# TODO: refactor the contrib libraries below this comment.

1
contrib/croaring vendored Submodule

@ -0,0 +1 @@
Subproject commit 5f20740ec0de5e153e8f4cb2ab91814e8b291a14

View File

@ -0,0 +1,25 @@
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/croaring)
set(SRCS
${LIBRARY_DIR}/src/array_util.c
${LIBRARY_DIR}/src/bitset_util.c
${LIBRARY_DIR}/src/containers/array.c
${LIBRARY_DIR}/src/containers/bitset.c
${LIBRARY_DIR}/src/containers/containers.c
${LIBRARY_DIR}/src/containers/convert.c
${LIBRARY_DIR}/src/containers/mixed_intersection.c
${LIBRARY_DIR}/src/containers/mixed_union.c
${LIBRARY_DIR}/src/containers/mixed_equal.c
${LIBRARY_DIR}/src/containers/mixed_subset.c
${LIBRARY_DIR}/src/containers/mixed_negation.c
${LIBRARY_DIR}/src/containers/mixed_xor.c
${LIBRARY_DIR}/src/containers/mixed_andnot.c
${LIBRARY_DIR}/src/containers/run.c
${LIBRARY_DIR}/src/roaring.c
${LIBRARY_DIR}/src/roaring_priority_queue.c
${LIBRARY_DIR}/src/roaring_array.c)
add_library(roaring ${SRCS})
target_include_directories(roaring PRIVATE ${LIBRARY_DIR}/include/roaring)
target_include_directories(roaring SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include)

View File

@ -1,6 +0,0 @@
add_library(roaring
roaring.c
roaring/roaring.h
roaring/roaring.hh)
target_include_directories (roaring SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})

View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016 The CRoaring authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,2 +0,0 @@
download from https://github.com/RoaringBitmap/CRoaring/archive/v0.2.57.tar.gz
and use ./amalgamation.sh generate

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -53,6 +53,7 @@ RUN apt-get update \
ninja-build \
psmisc \
python3 \
python3-pip \
python3-lxml \
python3-requests \
python3-termcolor \
@ -62,6 +63,8 @@ RUN apt-get update \
unixodbc \
--yes --no-install-recommends
RUN pip3 install numpy scipy pandas
# This symlink required by gcc to find lld compiler
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld

View File

@ -20,6 +20,7 @@ FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}")
FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}")
FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}")
FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}")
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
# Export these variables, so that all subsequent invocations of the script
# use them, and not try to guess them anew, which leads to weird effects.
@ -28,6 +29,7 @@ export FASTTEST_SOURCE
export FASTTEST_BUILD
export FASTTEST_DATA
export FASTTEST_OUT
export PATH
server_pid=none
@ -125,7 +127,7 @@ function clone_submodules
(
cd "$FASTTEST_SOURCE"
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11)
SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11 contrib/croaring)
git submodule sync
git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}"
@ -137,7 +139,14 @@ git submodule foreach git clean -xfd
function run_cmake
{
CMAKE_LIBS_CONFIG=("-DENABLE_LIBRARIES=0" "-DENABLE_TESTS=0" "-DENABLE_UTILS=0" "-DENABLE_EMBEDDED_COMPILER=0" "-DENABLE_THINLTO=0" "-DUSE_UNWIND=1")
CMAKE_LIBS_CONFIG=(
"-DENABLE_LIBRARIES=0"
"-DENABLE_TESTS=0"
"-DENABLE_UTILS=0"
"-DENABLE_EMBEDDED_COMPILER=0"
"-DENABLE_THINLTO=0"
"-DUSE_UNWIND=1"
)
# TODO remove this? we don't use ccache anyway. An option would be to download it
# from S3 simultaneously with cloning.
@ -259,7 +268,12 @@ TESTS_TO_SKIP=(
00974_query_profiler
# Look at DistributedFilesToInsert, so cannot run in parallel.
01460_DistributedFilesToInsert
01457_DistributedFilesToInsert
01541_max_memory_usage_for_user
# Require python libraries like scipy, pandas and numpy
01322_ttest_scipy
)
time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
@ -329,8 +343,6 @@ case "$stage" in
;&
"build")
build
PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH"
export PATH
;&
"configure")
# The `install_log.txt` is also needed for compatibility with old CI task --

View File

@ -9,6 +9,7 @@ RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
bash \
curl \
dmidecode \
g++ \
gdb \
git \
@ -37,7 +38,18 @@ RUN apt-get update \
COPY * /
CMD /entrypoint.sh
# Bind everything to one NUMA node, if there's more than one. Theoretically the
# node #0 should be less stable because of system interruptions. We bind
# randomly to node 1 or 0 to gather some statistics on that. We have to bind
# both servers and the tmpfs on which the database is stored. How to do it
# through Yandex Sandbox API is unclear, but by default tmpfs uses
# 'process allocation policy', not sure which process but hopefully the one that
# writes to it, so just bind the downloader script as well. We could also try to
# remount it with proper options in Sandbox task.
# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
# Double-escaped backslashes are a tribute to the engineering wonder of docker --
# it gives '/bin/sh: 1: [bash,: not found' otherwise.
CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"]
# docker run --network=host --volume <workspace>:/workspace --volume=<output>:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison

View File

@ -77,20 +77,33 @@ function restart
while killall clickhouse-server; do echo . ; sleep 1 ; done
echo all killed
set -m # Spawn servers in their own process groups
# Disable percpu arenas because they segfault when the process is bound to
# a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939
#
# About the jemalloc settings:
# https://github.com/jemalloc/jemalloc/wiki/Getting-Started
export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true"
left/clickhouse-server --config-file=left/config/config.xml -- --path left/db --user_files_path left/db/user_files &>> left-server-log.log &
set -m # Spawn servers in their own process groups
left/clickhouse-server --config-file=left/config/config.xml \
-- --path left/db --user_files_path left/db/user_files \
&>> left-server-log.log &
left_pid=$!
kill -0 $left_pid
disown $left_pid
right/clickhouse-server --config-file=right/config/config.xml -- --path right/db --user_files_path right/db/user_files &>> right-server-log.log &
right/clickhouse-server --config-file=right/config/config.xml \
-- --path right/db --user_files_path right/db/user_files \
&>> right-server-log.log &
right_pid=$!
kill -0 $right_pid
disown $right_pid
set +m
unset MALLOC_CONF
wait_for_server 9001 $left_pid
echo left ok
@ -449,7 +462,12 @@ wait
unset IFS
)
parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
# The comparison script might be bound to one NUMA node for better test
# stability, and the calculation runs out of memory because of this. Use
# all nodes.
numactl --show
numactl --cpunodebind=all --membind=all numactl --show
numactl --cpunodebind=all --membind=all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log
clickhouse-local --query "
-- Join the metric names back to the metric statistics we've calculated, and make
@ -1070,8 +1088,10 @@ case "$stage" in
time configure
;&
"restart")
numactl --show ||:
numactl --hardware ||:
lscpu ||:
dmidecode -t 4 ||:
time restart
;&
"run_tests")

View File

@ -14,6 +14,9 @@
we might also add time check to perf.py script.
-->
<max_execution_time>300</max_execution_time>
<!-- One NUMA node w/o hyperthreading -->
<max_threads>20</max_threads>
</default>
</profiles>
</yandex>

View File

@ -16,6 +16,7 @@ RUN apt-get update -y \
python3-lxml \
python3-requests \
python3-termcolor \
python3-pip \
qemu-user-static \
sudo \
telnet \
@ -23,6 +24,8 @@ RUN apt-get update -y \
unixodbc \
wget
RUN pip3 install numpy scipy pandas
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \

View File

@ -58,6 +58,7 @@ RUN apt-get --allow-unauthenticated update -y \
python3-lxml \
python3-requests \
python3-termcolor \
python3-pip \
qemu-user-static \
sudo \
telnet \
@ -68,6 +69,8 @@ RUN apt-get --allow-unauthenticated update -y \
wget \
zlib1g-dev
RUN pip3 install numpy scipy pandas
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \

View File

@ -195,7 +195,7 @@ Templates:
- [Function](_description_templates/template-function.md)
- [Setting](_description_templates/template-setting.md)
- [Table engine](_description_templates/template-table-engine.md)
- [Database or Table engine](_description_templates/template-engine.md)
- [System table](_description_templates/template-system-table.md)

View File

@ -1,8 +1,14 @@
# EngineName {#enginename}
- What the engine does.
- What the Database/Table engine does.
- Relations with other engines if they exist.
## Creating a Database {#creating-a-database}
``` sql
CREATE DATABASE ...
```
or
## Creating a Table {#creating-a-table}
``` sql
CREATE TABLE ...
@ -10,12 +16,19 @@
**Engine Parameters**
**Query Clauses**
**Query Clauses** (for Table engines only)
## Virtual columns {#virtual-columns}
## Virtual columns {#virtual-columns} (for Table engines only)
List and virtual columns with description, if they exist.
## Data Types Support {#data_types-support} (for Database engines only)
| EngineName | ClickHouse |
|-----------------------|------------------------------------|
| NativeDataTypeName | [ClickHouseDataTypeName](link#) |
## Specifics and recommendations {#specifics-and-recommendations}
Algorithms

View File

@ -47,6 +47,8 @@ select x; -- { serverError 49 }
```
This test ensures that the server returns an error with code 49 about unknown column `x`. If there is no error, or the error is different, the test will fail. If you want to ensure that an error occurs on the client side, use `clientError` annotation instead.
Do not check for a particular wording of error message, it may change in the future, and the test will needlessly break. Check only the error code. If the existing error code is not precise enough for your needs, consider adding a new one.
### Testing a Distributed Query
If you want to use distributed queries in functional tests, you can leverage `remote` table function with `127.0.0.{1..2}` addresses for the server to query itself; or you can use predefined test clusters in server configuration file like `test_shard_localhost`. Remember to add the words `shard` or `distributed` to the test name, so that it is ran in CI in correct configurations, where the server is configured to support distributed queries.

View File

@ -15,12 +15,83 @@ Returns a single `String`-type statement column, which contains a single v
## SHOW DATABASES {#show-databases}
``` sql
SHOW DATABASES [INTO OUTFILE filename] [FORMAT format]
Prints a list of all databases.
```sql
SHOW DATABASES [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
```
Prints a list of all databases.
This query is identical to `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`.
This statement is identical to the query:
```sql
SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
```
### Examples {#examples}
Getting database names, containing the symbols sequence 'de' in their names:
``` sql
SHOW DATABASES LIKE '%de%'
```
Result:
``` text
┌─name────┐
│ default │
└─────────┘
```
Getting database names, containing symbols sequence 'de' in their names, in the case insensitive manner:
``` sql
SHOW DATABASES ILIKE '%DE%'
```
Result:
``` text
┌─name────┐
│ default │
└─────────┘
```
Getting database names, not containing the symbols sequence 'de' in their names:
``` sql
SHOW DATABASES NOT LIKE '%de%'
```
Result:
``` text
┌─name───────────────────────────┐
│ _temporary_and_external_tables │
│ system │
│ test │
│ tutorial │
└────────────────────────────────┘
```
Getting the first two rows from database names:
``` sql
SHOW DATABASES LIMIT 2
```
Result:
``` text
┌─name───────────────────────────┐
│ _temporary_and_external_tables │
│ default │
└────────────────────────────────┘
```
### See Also {#see-also}
- [CREATE DATABASE](https://clickhouse.tech/docs/en/sql-reference/statements/create/database/#query-language-create-database)
## SHOW PROCESSLIST {#show-processlist}
@ -42,33 +113,86 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
Displays a list of tables.
``` sql
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE '<pattern>' | WHERE expr] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```sql
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```
If the `FROM` clause is not specified, the query returns the list of tables from the current database.
You can get the same results as the `SHOW TABLES` query in the following way:
This statement is identical to the query:
``` sql
SELECT name FROM system.tables WHERE database = <db> [AND name LIKE <pattern>] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```sql
SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```
**Example**
### Examples {#examples}
The following query selects the first two rows from the list of tables in the `system` database, whose names contain `co`.
Getting table names, containing the symbols sequence 'user' in their names:
``` sql
SHOW TABLES FROM system LIKE '%co%' LIMIT 2
SHOW TABLES FROM system LIKE '%user%'
```
Result:
``` text
┌─name─────────────┐
│ user_directories │
│ users │
└──────────────────┘
```
Getting table names, containing sequence 'user' in their names, in the case insensitive manner:
``` sql
SHOW TABLES FROM system ILIKE '%USER%'
```
Result:
``` text
┌─name─────────────┐
│ user_directories │
│ users │
└──────────────────┘
```
Getting table names, not containing the symbol sequence 's' in their names:
``` sql
SHOW TABLES FROM system NOT LIKE '%s%'
```
Result:
``` text
┌─name─────────┐
│ metric_log │
│ metric_log_0 │
│ metric_log_1 │
└──────────────┘
```
Getting the first two rows from table names:
``` sql
SHOW TABLES FROM system LIMIT 2
```
Result:
``` text
┌─name───────────────────────────┐
│ aggregate_function_combinators │
│ collations │
asynchronous_metric_log
└────────────────────────────────┘
```
### See Also {#see-also}
- [Create Tables](https://clickhouse.tech/docs/en/getting-started/tutorial/#create-tables)
- [SHOW CREATE TABLE](https://clickhouse.tech/docs/en/sql-reference/statements/show/#show-create-table)
## SHOW DICTIONARIES {#show-dictionaries}
Displays a list of [external dictionaries](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).

View File

@ -14,7 +14,7 @@
Репликация не зависит от шардирования. На каждом шарде репликация работает независимо.
Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../../engines/table-engines/mergetree-family/replication.md#query_language_queries_alter)).
Реплицируются сжатые данные запросов `INSERT`, `ALTER` (см. подробности в описании запроса [ALTER](../../../sql-reference/statements/alter/index.md#query_language_queries_alter)).
Запросы `CREATE`, `DROP`, `ATTACH`, `DETACH` и `RENAME` выполняются на одном сервере и не реплицируются:

View File

@ -10,12 +10,83 @@ SHOW CREATE [TEMPORARY] [TABLE|DICTIONARY] [db.]table [INTO OUTFILE filename] [F
## SHOW DATABASES {#show-databases}
``` sql
SHOW DATABASES [INTO OUTFILE filename] [FORMAT format]
Выводит список всех баз данных.
```sql
SHOW DATABASES [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
```
Выводит список всех баз данных.
Запрос полностью аналогичен запросу `SELECT name FROM system.databases [INTO OUTFILE filename] [FORMAT format]`.
Этот запрос идентичен запросу:
```sql
SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE filename] [FORMAT format]
```
### Примеры {#examples}
Получение списка баз данных, имена которых содержат последовательность символов 'de':
``` sql
SHOW DATABASES LIKE '%de%'
```
Результат:
``` text
┌─name────┐
│ default │
└─────────┘
```
Получение списка баз данных, имена которых содержат последовательность символов 'de' независимо от регистра:
``` sql
SHOW DATABASES ILIKE '%DE%'
```
Результат:
``` text
┌─name────┐
│ default │
└─────────┘
```
Получение списка баз данных, имена которых не содержат последовательность символов 'de':
``` sql
SHOW DATABASES NOT LIKE '%de%'
```
Результат:
``` text
┌─name───────────────────────────┐
│ _temporary_and_external_tables │
│ system │
│ test │
│ tutorial │
└────────────────────────────────┘
```
Получение первых двух строк из списка имен баз данных:
``` sql
SHOW DATABASES LIMIT 2
```
Результат:
``` text
┌─name───────────────────────────┐
│ _temporary_and_external_tables │
│ default │
└────────────────────────────────┘
```
### Смотрите также {#see-also}
- [CREATE DATABASE](https://clickhouse.tech/docs/ru/sql-reference/statements/create/database/#query-language-create-database)
## SHOW PROCESSLIST {#show-processlist}
@ -37,33 +108,86 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'"
Выводит список таблиц.
``` sql
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE '<pattern>' | WHERE expr] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```sql
SHOW [TEMPORARY] TABLES [{FROM | IN} <db>] [LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```
Если секция `FROM` не используется, то запрос возвращает список таблиц из текущей базы данных.
Если условие `FROM` не указано, запрос возвращает список таблиц из текущей базы данных.
Результат, идентичный тому, что выдаёт запрос `SHOW TABLES` можно получить также запросом следующего вида:
Этот запрос идентичен запросу:
``` sql
SELECT name FROM system.tables WHERE database = <db> [AND name LIKE <pattern>] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```sql
SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE '<pattern>'] [LIMIT <N>] [INTO OUTFILE <filename>] [FORMAT <format>]
```
**Пример**
### Примеры {#examples}
Следующий запрос выбирает первые две строки из списка таблиц в базе данных `system`, чьи имена содержат `co`.
Получение списка таблиц, имена которых содержат последовательность символов 'user':
``` sql
SHOW TABLES FROM system LIKE '%co%' LIMIT 2
SHOW TABLES FROM system LIKE '%user%'
```
Результат:
``` text
┌─name─────────────┐
│ user_directories │
│ users │
└──────────────────┘
```
Получение списка таблиц, имена которых содержат последовательность символов 'user' без учета регистра:
``` sql
SHOW TABLES FROM system ILIKE '%USER%'
```
Результат:
``` text
┌─name─────────────┐
│ user_directories │
│ users │
└──────────────────┘
```
Получение списка таблиц, имена которых не содержат символ 's':
``` sql
SHOW TABLES FROM system NOT LIKE '%s%'
```
Результат:
``` text
┌─name─────────┐
│ metric_log │
│ metric_log_0 │
│ metric_log_1 │
└──────────────┘
```
Получение первых двух строк из списка таблиц:
``` sql
SHOW TABLES FROM system LIMIT 2
```
Результат:
``` text
┌─name───────────────────────────┐
│ aggregate_function_combinators │
│ collations │
asynchronous_metric_log
└────────────────────────────────┘
```
### Смотрите также {#see-also}
- [Create Tables](https://clickhouse.tech/docs/ru/getting-started/tutorial/#create-tables)
- [SHOW CREATE TABLE](https://clickhouse.tech/docs/ru/sql-reference/statements/show/#show-create-table)
## SHOW DICTIONARIES {#show-dictionaries}
Выводит список [внешних словарей](../../sql-reference/statements/show.md).

View File

@ -1896,7 +1896,7 @@ private:
if (has_vertical_output_suffix)
throw Exception("Output format already specified", ErrorCodes::CLIENT_OUTPUT_FORMAT_SPECIFIED);
const auto & id = query_with_output->format->as<ASTIdentifier &>();
current_format = id.name;
current_format = id.name();
}
}

View File

@ -168,11 +168,11 @@ ASTPtr extractOrderBy(const ASTPtr & storage_ast)
throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS);
}
/// Wraps only identifiers with backticks.
/// Wraps only identifiers with backticks.
std::string wrapIdentifiersWithBackticks(const ASTPtr & root)
{
if (auto identifier = std::dynamic_pointer_cast<ASTIdentifier>(root))
return backQuote(identifier->name);
return backQuote(identifier->name());
if (auto function = std::dynamic_pointer_cast<ASTFunction>(root))
return function->name + '(' + wrapIdentifiersWithBackticks(function->arguments) + ')';
@ -214,7 +214,7 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast)
for (size_t i = 0; i < sorting_key_size; ++i)
{
/// Column name could be represented as a f_1(f_2(...f_n(column_name))).
/// Each f_i could take one or more parameters.
/// Each f_i could take one or more parameters.
/// We will wrap identifiers with backticks to allow non-standart identifier names.
String sorting_key_column = sorting_key_expr_list->children[i]->getColumnName();

View File

@ -258,7 +258,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
Poco::Logger * log = &logger();
UseSSL use_ssl;
ThreadStatus thread_status;
MainThreadStatus::getInstance();
registerFunctions();
registerAggregateFunctions();

View File

@ -392,6 +392,22 @@
</replica>
</shard>
</test_cluster_two_shards>
<test_cluster_two_shards_internal_replication>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>127.0.0.2</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_two_shards_internal_replication>
<test_shard_localhost_secure>
<shard>
<replica>

View File

@ -13,7 +13,6 @@
// this one: https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/roaring.h
#include <roaring/roaring.h>
namespace DB
{
/**
@ -599,128 +598,6 @@ public:
}
}
private:
/// To read and write the DB Buffer directly, migrate code from CRoaring
void db_roaring_bitmap_add_many(DB::ReadBuffer & db_buf, roaring_bitmap_t * r, size_t n_args)
{
void * container = nullptr; // hold value of last container touched
uint8_t typecode = 0; // typecode of last container touched
uint32_t prev = 0; // previous valued inserted
size_t i = 0; // index of value
int containerindex = 0;
if (n_args == 0)
return;
uint32_t val;
readBinary(val, db_buf);
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
prev = val;
++i;
for (; i < n_args; ++i)
{
readBinary(val, db_buf);
if (((prev ^ val) >> 16) == 0)
{ // no need to seek the container, it is at hand
// because we already have the container at hand, we can do the
// insertion
// automatically, bypassing the roaring_bitmap_add call
uint8_t newtypecode = typecode;
void * container2 = container_add(container, val & 0xFFFF, typecode, &newtypecode);
// rare instance when we need to
if (container2 != container)
{
// change the container type
container_free(container, typecode);
ra_set_container_at_index(&r->high_low_container, containerindex, container2, newtypecode);
typecode = newtypecode;
container = container2;
}
}
else
{
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
}
prev = val;
}
}
void db_ra_to_uint32_array(DB::WriteBuffer & db_buf, roaring_array_t * ra) const
{
size_t ctr = 0;
for (Int32 i = 0; i < ra->size; ++i)
{
Int32 num_added = db_container_to_uint32_array(db_buf, ra->containers[i], ra->typecodes[i], (static_cast<UInt32>(ra->keys[i])) << 16);
ctr += num_added;
}
}
UInt32 db_container_to_uint32_array(DB::WriteBuffer & db_buf, const void * container, uint8_t typecode, UInt32 base) const
{
container = container_unwrap_shared(container, &typecode);
switch (typecode)
{
case BITSET_CONTAINER_TYPE_CODE:
return db_bitset_container_to_uint32_array(db_buf, static_cast<const bitset_container_t *>(container), base);
case ARRAY_CONTAINER_TYPE_CODE:
return db_array_container_to_uint32_array(db_buf, static_cast<const array_container_t *>(container), base);
case RUN_CONTAINER_TYPE_CODE:
return db_run_container_to_uint32_array(db_buf, static_cast<const run_container_t *>(container), base);
}
return 0;
}
UInt32 db_bitset_container_to_uint32_array(DB::WriteBuffer & db_buf, const bitset_container_t * cont, UInt32 base) const
{
return static_cast<UInt32>(db_bitset_extract_setbits(db_buf, cont->array, BITSET_CONTAINER_SIZE_IN_WORDS, base));
}
size_t db_bitset_extract_setbits(DB::WriteBuffer & db_buf, UInt64 * bitset, size_t length, UInt32 base) const
{
UInt32 outpos = 0;
for (size_t i = 0; i < length; ++i)
{
UInt64 w = bitset[i];
while (w != 0)
{
UInt64 t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail)
UInt32 r = __builtin_ctzll(w); // on x64, should compile to TZCNT
UInt32 val = r + base;
writePODBinary(val, db_buf);
outpos++;
w ^= t;
}
base += 64;
}
return outpos;
}
int db_array_container_to_uint32_array(DB::WriteBuffer & db_buf, const array_container_t * cont, UInt32 base) const
{
UInt32 outpos = 0;
for (Int32 i = 0; i < cont->cardinality; ++i)
{
const UInt32 val = base + cont->array[i];
writePODBinary(val, db_buf);
outpos++;
}
return outpos;
}
int db_run_container_to_uint32_array(DB::WriteBuffer & db_buf, const run_container_t * cont, UInt32 base) const
{
UInt32 outpos = 0;
for (Int32 i = 0; i < cont->n_runs; ++i)
{
UInt32 run_start = base + cont->runs[i].value;
UInt16 le = cont->runs[i].length;
for (Int32 j = 0; j <= le; ++j)
{
UInt32 val = run_start + j;
writePODBinary(val, db_buf);
outpos++;
}
}
return outpos;
}
};
template <typename T>

View File

@ -0,0 +1,52 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionStudentTTest.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include "registerAggregateFunctions.h"
#include <AggregateFunctions/Helpers.h>
#include <DataTypes/DataTypeAggregateFunction.h>
// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse)
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
namespace DB
{
namespace
{
AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters)
{
assertBinary(name, argument_types);
assertNoParameters(name, parameters);
AggregateFunctionPtr res;
if (isDecimal(argument_types[0]) || isDecimal(argument_types[1]))
{
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
}
else
{
res.reset(createWithTwoNumericTypes<AggregateFunctionStudentTTest>(*argument_types[0], *argument_types[1], argument_types));
}
if (!res)
{
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
}
return res;
}
}
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory)
{
factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive);
}
}

View File

@ -0,0 +1,253 @@
#pragma once
#include <AggregateFunctions/IAggregateFunction.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnTuple.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitors.h>
#include <Core/Types.h>
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <limits>
#include <cmath>
#include <functional>
#include <type_traits>
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
namespace DB
{
template <typename X = Float64, typename Y = Float64>
struct AggregateFunctionStudentTTestData final
{
size_t size_x = 0;
size_t size_y = 0;
X sum_x = static_cast<X>(0);
Y sum_y = static_cast<Y>(0);
X square_sum_x = static_cast<X>(0);
Y square_sum_y = static_cast<Y>(0);
Float64 mean_x = static_cast<Float64>(0);
Float64 mean_y = static_cast<Float64>(0);
void add(X x, Y y)
{
sum_x += x;
sum_y += y;
size_x++;
size_y++;
mean_x = static_cast<Float64>(sum_x) / size_x;
mean_y = static_cast<Float64>(sum_y) / size_y;
square_sum_x += x * x;
square_sum_y += y * y;
}
void merge(const AggregateFunctionStudentTTestData &other)
{
sum_x += other.sum_x;
sum_y += other.sum_y;
size_x += other.size_x;
size_y += other.size_y;
mean_x = static_cast<Float64>(sum_x) / size_x;
mean_y = static_cast<Float64>(sum_y) / size_y;
square_sum_x += other.square_sum_x;
square_sum_y += other.square_sum_y;
}
void serialize(WriteBuffer &buf) const
{
writeBinary(mean_x, buf);
writeBinary(mean_y, buf);
writeBinary(sum_x, buf);
writeBinary(sum_y, buf);
writeBinary(square_sum_x, buf);
writeBinary(square_sum_y, buf);
writeBinary(size_x, buf);
writeBinary(size_y, buf);
}
void deserialize(ReadBuffer &buf)
{
readBinary(mean_x, buf);
readBinary(mean_y, buf);
readBinary(sum_x, buf);
readBinary(sum_y, buf);
readBinary(square_sum_x, buf);
readBinary(square_sum_y, buf);
readBinary(size_x, buf);
readBinary(size_y, buf);
}
size_t getSizeY() const
{
return size_y;
}
size_t getSizeX() const
{
return size_x;
}
Float64 getSSquared() const
{
/// The original formulae looks like
/// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2}
/// But we made some mathematical transformations not to store original sequences.
/// Also we dropped sqrt, because later it will be squared later.
const Float64 all_x = square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x;
const Float64 all_y = square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y;
return static_cast<Float64>(all_x + all_y) / (size_x + size_y - 2);
}
Float64 getTStatisticSquared() const
{
return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared();
}
Float64 getTStatistic() const
{
return (mean_x - mean_y) / std::sqrt(getStandartErrorSquared());
}
Float64 getStandartErrorSquared() const
{
if (size_x == 0 || size_y == 0)
throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS);
return getSSquared() * (1.0 / static_cast<Float64>(size_x) + 1.0 / static_cast<Float64>(size_y));
}
Float64 getDegreesOfFreedom() const
{
return static_cast<Float64>(size_x + size_y - 2);
}
static Float64 integrateSimpson(Float64 a, Float64 b, std::function<Float64(Float64)> func)
{
const size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b)));
const long double h = (b - a) / iterations;
Float64 sum_odds = 0.0;
for (size_t i = 1; i < iterations; i += 2)
sum_odds += func(a + i * h);
Float64 sum_evens = 0.0;
for (size_t i = 2; i < iterations; i += 2)
sum_evens += func(a + i * h);
return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3;
}
Float64 getPValue() const
{
const Float64 v = getDegreesOfFreedom();
const Float64 t = getTStatisticSquared();
auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); };
Float64 numenator = integrateSimpson(0, v / (t + v), f);
Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5));
return numenator / denominator;
}
std::pair<Float64, Float64> getResult() const
{
return std::make_pair(getTStatistic(), getPValue());
}
};
/// Returns tuple of (t-statistic, p-value)
/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf
template <typename X = Float64, typename Y = Float64>
class AggregateFunctionStudentTTest :
public IAggregateFunctionDataHelper<AggregateFunctionStudentTTestData<X, Y>,AggregateFunctionStudentTTest<X, Y>>
{
public:
AggregateFunctionStudentTTest(const DataTypes & arguments)
: IAggregateFunctionDataHelper<AggregateFunctionStudentTTestData<X, Y>, AggregateFunctionStudentTTest<X, Y>> ({arguments}, {})
{}
String getName() const override
{
return "studentTTest";
}
DataTypePtr getReturnType() const override
{
DataTypes types
{
std::make_shared<DataTypeNumber<Float64>>(),
std::make_shared<DataTypeNumber<Float64>>(),
};
Strings names
{
"t-statistic",
"p-value"
};
return std::make_shared<DataTypeTuple>(
std::move(types),
std::move(names)
);
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
auto col_x = assert_cast<const ColumnVector<X> *>(columns[0]);
auto col_y = assert_cast<const ColumnVector<Y> *>(columns[1]);
X x = col_x->getData()[row_num];
Y y = col_y->getData()[row_num];
this->data(place).add(x, y);
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).merge(this->data(rhs));
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).serialize(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).deserialize(buf);
}
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override
{
size_t size_x = this->data(place).getSizeX();
size_t size_y = this->data(place).getSizeY();
if (size_x < 2 || size_y < 2)
{
throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS);
}
Float64 t_statistic = 0.0;
Float64 p_value = 0.0;
std::tie(t_statistic, p_value) = this->data(place).getResult();
/// Because p-value is a probability.
p_value = std::min(1.0, std::max(0.0, p_value));
auto & column_tuple = assert_cast<ColumnTuple &>(to);
auto & column_stat = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(0));
auto & column_value = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(1));
column_stat.getData().push_back(t_statistic);
column_value.getData().push_back(p_value);
}
};
};

View File

@ -92,7 +92,7 @@ struct AggregateFunctionTimeSeriesGroupSumData
it_ss->second.add(t, v);
}
if (result.size() > 0 && t < result.back().first)
throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc!!!", ErrorCodes::LOGICAL_ERROR};
throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc.", ErrorCodes::LOGICAL_ERROR};
if (result.size() > 0 && t == result.back().first)
{
//do not add new point

View File

@ -0,0 +1,49 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionWelchTTest.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include "registerAggregateFunctions.h"
#include <AggregateFunctions/Helpers.h>
#include <DataTypes/DataTypeAggregateFunction.h>
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
namespace DB
{
namespace
{
AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters)
{
assertBinary(name, argument_types);
assertNoParameters(name, parameters);
AggregateFunctionPtr res;
if (isDecimal(argument_types[0]) || isDecimal(argument_types[1]))
{
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
}
else
{
res.reset(createWithTwoNumericTypes<AggregateFunctionWelchTTest>(*argument_types[0], *argument_types[1], argument_types));
}
if (!res)
{
throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED);
}
return res;
}
}
void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory)
{
factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive);
}
}

View File

@ -0,0 +1,264 @@
#pragma once
#include <AggregateFunctions/IAggregateFunction.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnTuple.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitors.h>
#include <Core/Types.h>
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <limits>
#include <cmath>
#include <functional>
#include <type_traits>
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
namespace DB
{
template <typename X = Float64, typename Y = Float64>
struct AggregateFunctionWelchTTestData final
{
size_t size_x = 0;
size_t size_y = 0;
X sum_x = static_cast<X>(0);
Y sum_y = static_cast<Y>(0);
X square_sum_x = static_cast<X>(0);
Y square_sum_y = static_cast<Y>(0);
Float64 mean_x = static_cast<Float64>(0);
Float64 mean_y = static_cast<Float64>(0);
void add(X x, Y y)
{
sum_x += x;
sum_y += y;
size_x++;
size_y++;
mean_x = static_cast<Float64>(sum_x) / size_x;
mean_y = static_cast<Float64>(sum_y) / size_y;
square_sum_x += x * x;
square_sum_y += y * y;
}
void merge(const AggregateFunctionWelchTTestData &other)
{
sum_x += other.sum_x;
sum_y += other.sum_y;
size_x += other.size_x;
size_y += other.size_y;
mean_x = static_cast<Float64>(sum_x) / size_x;
mean_y = static_cast<Float64>(sum_y) / size_y;
square_sum_x += other.square_sum_x;
square_sum_y += other.square_sum_y;
}
void serialize(WriteBuffer &buf) const
{
writeBinary(mean_x, buf);
writeBinary(mean_y, buf);
writeBinary(sum_x, buf);
writeBinary(sum_y, buf);
writeBinary(square_sum_x, buf);
writeBinary(square_sum_y, buf);
writeBinary(size_x, buf);
writeBinary(size_y, buf);
}
void deserialize(ReadBuffer &buf)
{
readBinary(mean_x, buf);
readBinary(mean_y, buf);
readBinary(sum_x, buf);
readBinary(sum_y, buf);
readBinary(square_sum_x, buf);
readBinary(square_sum_y, buf);
readBinary(size_x, buf);
readBinary(size_y, buf);
}
size_t getSizeY() const
{
return size_y;
}
size_t getSizeX() const
{
return size_x;
}
Float64 getSxSquared() const
{
/// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2}
/// But we made some mathematical transformations not to store original sequences.
/// Also we dropped sqrt, because later it will be squared later.
return static_cast<Float64>(square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x) / (size_x - 1);
}
Float64 getSySquared() const
{
/// The original formulae looks like \frac{1}{size_y - 1} \sum_{i = 1}^{size_y}{(y_i - \bar{y}) ^ 2}
/// But we made some mathematical transformations not to store original sequences.
/// Also we dropped sqrt, because later it will be squared later.
return static_cast<Float64>(square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y) / (size_y - 1);
}
Float64 getTStatisticSquared() const
{
if (size_x == 0 || size_y == 0)
{
throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS);
}
return std::pow(mean_x - mean_y, 2) / (getSxSquared() / size_x + getSySquared() / size_y);
}
Float64 getTStatistic() const
{
if (size_x == 0 || size_y == 0)
{
throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS);
}
return (mean_x - mean_y) / std::sqrt(getSxSquared() / size_x + getSySquared() / size_y);
}
Float64 getDegreesOfFreedom() const
{
auto sx = getSxSquared();
auto sy = getSySquared();
Float64 numerator = std::pow(sx / size_x + sy / size_y, 2);
Float64 denominator_first = std::pow(sx, 2) / (std::pow(size_x, 2) * (size_x - 1));
Float64 denominator_second = std::pow(sy, 2) / (std::pow(size_y, 2) * (size_y - 1));
return numerator / (denominator_first + denominator_second);
}
static Float64 integrateSimpson(Float64 a, Float64 b, std::function<Float64(Float64)> func)
{
size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b)));
double h = (b - a) / iterations;
Float64 sum_odds = 0.0;
for (size_t i = 1; i < iterations; i += 2)
sum_odds += func(a + i * h);
Float64 sum_evens = 0.0;
for (size_t i = 2; i < iterations; i += 2)
sum_evens += func(a + i * h);
return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3;
}
Float64 getPValue() const
{
const Float64 v = getDegreesOfFreedom();
const Float64 t = getTStatisticSquared();
auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); };
Float64 numenator = integrateSimpson(0, v / (t + v), f);
Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5));
return numenator / denominator;
}
std::pair<Float64, Float64> getResult() const
{
return std::make_pair(getTStatistic(), getPValue());
}
};
/// Returns tuple of (t-statistic, p-value)
/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf
template <typename X = Float64, typename Y = Float64>
class AggregateFunctionWelchTTest :
public IAggregateFunctionDataHelper<AggregateFunctionWelchTTestData<X, Y>,AggregateFunctionWelchTTest<X, Y>>
{
public:
AggregateFunctionWelchTTest(const DataTypes & arguments)
: IAggregateFunctionDataHelper<AggregateFunctionWelchTTestData<X, Y>, AggregateFunctionWelchTTest<X, Y>> ({arguments}, {})
{}
String getName() const override
{
return "welchTTest";
}
DataTypePtr getReturnType() const override
{
DataTypes types
{
std::make_shared<DataTypeNumber<Float64>>(),
std::make_shared<DataTypeNumber<Float64>>(),
};
Strings names
{
"t-statistic",
"p-value"
};
return std::make_shared<DataTypeTuple>(
std::move(types),
std::move(names)
);
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
auto col_x = assert_cast<const ColumnVector<X> *>(columns[0]);
auto col_y = assert_cast<const ColumnVector<Y> *>(columns[1]);
X x = col_x->getData()[row_num];
Y y = col_y->getData()[row_num];
this->data(place).add(x, y);
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).merge(this->data(rhs));
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).serialize(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).deserialize(buf);
}
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override
{
size_t size_x = this->data(place).getSizeX();
size_t size_y = this->data(place).getSizeY();
if (size_x < 2 || size_y < 2)
{
throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS);
}
Float64 t_statistic = 0.0;
Float64 p_value = 0.0;
std::tie(t_statistic, p_value) = this->data(place).getResult();
/// Because p-value is a probability.
p_value = std::min(1.0, std::max(0.0, p_value));
auto & column_tuple = assert_cast<ColumnTuple &>(to);
auto & column_stat = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(0));
auto & column_value = assert_cast<ColumnVector<Float64> &>(column_tuple.getColumn(1));
column_stat.getData().push_back(t_statistic);
column_value.getData().push_back(p_value);
}
};
};

View File

@ -45,6 +45,8 @@ void registerAggregateFunctions()
registerAggregateFunctionMoving(factory);
registerAggregateFunctionCategoricalIV(factory);
registerAggregateFunctionAggThrow(factory);
registerAggregateFunctionWelchTTest(factory);
registerAggregateFunctionStudentTTest(factory);
registerAggregateFunctionRankCorrelation(factory);
}

View File

@ -35,6 +35,8 @@ void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &)
void registerAggregateFunctionMoving(AggregateFunctionFactory &);
void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &);
void registerAggregateFunctionAggThrow(AggregateFunctionFactory &);
void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &);
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &);
void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory &);
class AggregateFunctionCombinatorFactory;

View File

@ -42,6 +42,7 @@ SRCS(
AggregateFunctionState.cpp
AggregateFunctionStatistics.cpp
AggregateFunctionStatisticsSimple.cpp
AggregateFunctionStudentTTest.cpp
AggregateFunctionSum.cpp
AggregateFunctionSumMap.cpp
AggregateFunctionTimeSeriesGroupSum.cpp
@ -49,6 +50,7 @@ SRCS(
AggregateFunctionUniqCombined.cpp
AggregateFunctionUniq.cpp
AggregateFunctionUniqUpTo.cpp
AggregateFunctionWelchTTest.cpp
AggregateFunctionWindowFunnel.cpp
parseAggregateFunctionParameters.cpp
registerAggregateFunctions.cpp

View File

@ -173,14 +173,20 @@ add_object_library(clickhouse_processors_merges Processors/Merges)
add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Algorithms)
add_object_library(clickhouse_processors_queryplan Processors/QueryPlan)
set (DBMS_COMMON_LIBRARIES)
# libgcc_s does not provide an implementation of an atomics library. Instead,
# GCCs libatomic library can be used to supply these when using libgcc_s.
if ((NOT USE_LIBCXX) AND COMPILER_CLANG AND OS_LINUX)
list (APPEND DBMS_COMMON_LIBRARIES atomic)
endif()
if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
add_library (dbms STATIC ${dbms_headers} ${dbms_sources})
target_link_libraries (dbms PRIVATE jemalloc libdivide)
target_link_libraries (dbms PRIVATE jemalloc libdivide ${DBMS_COMMON_LIBRARIES})
set (all_modules dbms)
else()
add_library (dbms SHARED ${dbms_headers} ${dbms_sources})
target_link_libraries (dbms PUBLIC ${all_modules})
target_link_libraries (dbms PUBLIC ${all_modules} ${DBMS_COMMON_LIBRARIES})
target_link_libraries (clickhouse_interpreters PRIVATE jemalloc libdivide)
list (APPEND all_modules dbms)
# force all split libs to be linked

View File

@ -18,7 +18,7 @@ public:
void ALWAYS_INLINE forEachMapped(Func && func)
{
for (auto i = 0u; i < this->NUM_BUCKETS; ++i)
return this->impls[i].forEachMapped(func);
this->impls[i].forEachMapped(func);
}
TMapped & ALWAYS_INLINE operator[](const Key & x)

View File

@ -13,6 +13,24 @@
#include <random>
#include <cstdlib>
namespace
{
MemoryTracker * getMemoryTracker()
{
if (auto * thread_memory_tracker = DB::CurrentThread::getMemoryTracker())
return thread_memory_tracker;
/// Once the main thread is initialized,
/// total_memory_tracker is initialized too.
/// And can be used, since MainThreadStatus is required for profiling.
if (DB::MainThreadStatus::get())
return &total_memory_tracker;
return nullptr;
}
}
namespace DB
{
@ -192,14 +210,15 @@ void MemoryTracker::free(Int64 size)
DB::TraceCollector::collect(DB::TraceType::MemorySample, StackTrace(), -size);
}
Int64 accounted_size = size;
if (level == VariableContext::Thread)
{
/// Could become negative if memory allocated in this thread is freed in another one
amount.fetch_sub(size, std::memory_order_relaxed);
amount.fetch_sub(accounted_size, std::memory_order_relaxed);
}
else
{
Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
Int64 new_amount = amount.fetch_sub(accounted_size, std::memory_order_relaxed) - accounted_size;
/** Sometimes, query could free some data, that was allocated outside of query context.
* Example: cache eviction.
@ -210,7 +229,7 @@ void MemoryTracker::free(Int64 size)
if (unlikely(new_amount < 0))
{
amount.fetch_sub(new_amount);
size += new_amount;
accounted_size += new_amount;
}
}
@ -218,7 +237,7 @@ void MemoryTracker::free(Int64 size)
loaded_next->free(size);
if (metric != CurrentMetrics::end())
CurrentMetrics::sub(metric, size);
CurrentMetrics::sub(metric, accounted_size);
}
@ -270,16 +289,24 @@ namespace CurrentMemoryTracker
void alloc(Int64 size)
{
if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker())
if (auto * memory_tracker = getMemoryTracker())
{
current_thread->untracked_memory += size;
if (current_thread->untracked_memory > current_thread->untracked_memory_limit)
if (current_thread)
{
/// Zero untracked before track. If tracker throws out-of-limit we would be able to alloc up to untracked_memory_limit bytes
/// more. It could be useful to enlarge Exception message in rethrow logic.
Int64 tmp = current_thread->untracked_memory;
current_thread->untracked_memory = 0;
memory_tracker->alloc(tmp);
current_thread->untracked_memory += size;
if (current_thread->untracked_memory > current_thread->untracked_memory_limit)
{
/// Zero untracked before track. If tracker throws out-of-limit we would be able to alloc up to untracked_memory_limit bytes
/// more. It could be useful to enlarge Exception message in rethrow logic.
Int64 tmp = current_thread->untracked_memory;
current_thread->untracked_memory = 0;
memory_tracker->alloc(tmp);
}
}
/// total_memory_tracker only, ignore untracked_memory
else
{
memory_tracker->alloc(size);
}
}
}
@ -292,13 +319,21 @@ namespace CurrentMemoryTracker
void free(Int64 size)
{
if (auto * memory_tracker = DB::CurrentThread::getMemoryTracker())
if (auto * memory_tracker = getMemoryTracker())
{
current_thread->untracked_memory -= size;
if (current_thread->untracked_memory < -current_thread->untracked_memory_limit)
if (current_thread)
{
memory_tracker->free(-current_thread->untracked_memory);
current_thread->untracked_memory = 0;
current_thread->untracked_memory -= size;
if (current_thread->untracked_memory < -current_thread->untracked_memory_limit)
{
memory_tracker->free(-current_thread->untracked_memory);
current_thread->untracked_memory = 0;
}
}
/// total_memory_tracker only, ignore untracked_memory
else
{
memory_tracker->free(size);
}
}
}

View File

@ -20,6 +20,7 @@ namespace ErrorCodes
thread_local ThreadStatus * current_thread = nullptr;
thread_local ThreadStatus * main_thread = nullptr;
ThreadStatus::ThreadStatus()
@ -115,4 +116,20 @@ void ThreadStatus::onFatalError()
fatal_error_callback();
}
ThreadStatus * MainThreadStatus::main_thread = nullptr;
MainThreadStatus & MainThreadStatus::getInstance()
{
static MainThreadStatus thread_status;
return thread_status;
}
MainThreadStatus::MainThreadStatus()
: ThreadStatus()
{
main_thread = current_thread;
}
MainThreadStatus::~MainThreadStatus()
{
main_thread = nullptr;
}
}

View File

@ -215,4 +215,22 @@ private:
void setupState(const ThreadGroupStatusPtr & thread_group_);
};
/**
* Creates ThreadStatus for the main thread.
*/
class MainThreadStatus : public ThreadStatus
{
public:
static MainThreadStatus & getInstance();
static ThreadStatus * get() { return main_thread; }
static bool isMainThread() { return main_thread == current_thread; }
~MainThreadStatus();
private:
MainThreadStatus();
static ThreadStatus * main_thread;
};
}

View File

@ -66,10 +66,20 @@ void TraceCollector::collect(TraceType trace_type, const StackTrace & stack_trac
char buffer[buf_size];
WriteBufferFromFileDescriptorDiscardOnFailure out(pipe.fds_rw[1], buf_size, buffer);
StringRef query_id = CurrentThread::getQueryId();
query_id.size = std::min(query_id.size, QUERY_ID_MAX_LEN);
StringRef query_id;
UInt64 thread_id;
auto thread_id = CurrentThread::get().thread_id;
if (CurrentThread::isInitialized())
{
query_id = CurrentThread::getQueryId();
query_id.size = std::min(query_id.size, QUERY_ID_MAX_LEN);
thread_id = CurrentThread::get().thread_id;
}
else
{
thread_id = MainThreadStatus::get()->thread_id;
}
writeChar(false, out); /// true if requested to stop the collecting thread.
writeStringBinary(query_id, out);

View File

@ -76,7 +76,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr
ASTPtr codec_arguments;
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
{
codec_family_name = family_name->name;
codec_family_name = family_name->name();
codec_arguments = {};
}
else if (const auto * ast_func = inner_codec_ast->as<ASTFunction>())
@ -207,7 +207,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IData
ASTPtr codec_arguments;
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
{
codec_family_name = family_name->name;
codec_family_name = family_name->name();
codec_arguments = {};
}
else if (const auto * ast_func = inner_codec_ast->as<ASTFunction>())

View File

@ -5,9 +5,6 @@ target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLU
add_executable (field field.cpp)
target_link_libraries (field PRIVATE dbms)
add_executable (move_field move_field.cpp)
target_link_libraries (move_field PRIVATE clickhouse_common_io)
add_executable (string_ref_hash string_ref_hash.cpp)
target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)

View File

@ -0,0 +1,22 @@
#include <gtest/gtest.h>
#include <Core/Field.h>
using namespace DB;
GTEST_TEST(Field, Move)
{
Field f;
f = Field{String{"Hello, world (1)"}};
ASSERT_EQ(f.get<String>(), "Hello, world (1)");
f = Field{String{"Hello, world (2)"}};
ASSERT_EQ(f.get<String>(), "Hello, world (2)");
f = Field{Array{Field{String{"Hello, world (3)"}}}};
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (3)");
f = String{"Hello, world (4)"};
ASSERT_EQ(f.get<String>(), "Hello, world (4)");
f = Array{Field{String{"Hello, world (5)"}}};
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (5)");
f = Array{String{"Hello, world (6)"}};
ASSERT_EQ(f.get<Array>()[0].get<String>(), "Hello, world (6)");
}

View File

@ -1,25 +0,0 @@
#include <iostream>
#include <Core/Field.h>
int main(int, char **)
{
using namespace DB;
Field f;
f = Field{String{"Hello, world"}};
std::cerr << f.get<String>() << "\n";
f = Field{String{"Hello, world!"}};
std::cerr << f.get<String>() << "\n";
f = Field{Array{Field{String{"Hello, world!!"}}}};
std::cerr << f.get<Array>()[0].get<String>() << "\n";
f = String{"Hello, world!!!"};
std::cerr << f.get<String>() << "\n";
f = Array{Field{String{"Hello, world!!!!"}}};
std::cerr << f.get<Array>()[0].get<String>() << "\n";
f = Array{String{"Hello, world!!!!!"}};
std::cerr << f.get<Array>()[0].get<String>() << "\n";
return 0;
}

View File

@ -43,7 +43,7 @@ DataTypePtr DataTypeFactory::get(const ASTPtr & ast) const
if (const auto * ident = ast->as<ASTIdentifier>())
{
return get(ident->name, {});
return get(ident->name(), {});
}
if (const auto * lit = ast->as<ASTLiteral>())

View File

@ -384,7 +384,7 @@ static DataTypePtr create(const ASTPtr & arguments)
throw Exception("String data type family mustn't have more than one argument - size in characters", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
const auto * argument = arguments->children[0]->as<ASTLiteral>();
if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get<UInt64>() == 0)
if (!argument || argument->value.getType() != Field::Types::UInt64)
throw Exception("String data type family may have only a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
}

View File

@ -329,4 +329,10 @@ const StoragePtr & DatabaseLazyIterator::table() const
return current_storage;
}
void DatabaseLazyIterator::reset()
{
if (current_storage)
current_storage.reset();
}
}

View File

@ -122,6 +122,7 @@ public:
bool isValid() const override;
const String & name() const override;
const StoragePtr & table() const override;
void reset() override;
private:
const DatabaseLazy & database;

View File

@ -44,6 +44,8 @@ public:
/// (a database with support for lazy tables loading
/// - it maintains a list of tables but tables are loaded lazily).
virtual const StoragePtr & table() const = 0;
/// Reset reference counter to the StoragePtr.
virtual void reset() = 0;
virtual ~IDatabaseTablesIterator() = default;
@ -93,6 +95,8 @@ public:
const String & name() const override { return it->first; }
const StoragePtr & table() const override { return it->second; }
void reset() override { it->second.reset(); }
};
/// Copies list of dictionaries and iterates through such snapshot.

View File

@ -28,6 +28,11 @@ public:
return tables.emplace_back(storage);
}
void reset() override
{
tables.clear();
}
UUID uuid() const override { return nested_iterator->uuid(); }
DatabaseMaterializeTablesIterator(DatabaseTablesIteratorPtr nested_iterator_, DatabaseMaterializeMySQL * database_)

View File

@ -172,7 +172,7 @@ Names getPrimaryKeyColumns(const ASTExpressionList * primary_key)
for (size_t index = 0; index != children.size(); ++index)
{
const ASTIdentifier * key_part = children[index]->as<const ASTIdentifier>();
result.push_back(key_part->name);
result.push_back(key_part->name());
}
return result;
}
@ -367,7 +367,7 @@ void buildConfigurationFromFunctionWithKeyValueArguments(
if (const auto * identifier = pair->second->as<const ASTIdentifier>(); identifier)
{
AutoPtr<Text> value(doc->createTextNode(identifier->name));
AutoPtr<Text> value(doc->createTextNode(identifier->name()));
current_xml_element->appendChild(value);
}
else if (const auto * literal = pair->second->as<const ASTLiteral>(); literal)

View File

@ -307,11 +307,6 @@ private:
}
const auto input_value = input_column->getDataAt(r);
auto aad_value = StringRef{};
if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM && !std::is_same_v<std::nullptr_t, std::decay_t<AadColumnType>>)
{
aad_value = aad_column->getDataAt(r);
}
if constexpr (mode != CipherMode::MySQLCompatibility)
{

View File

@ -82,13 +82,13 @@ public:
std::shared_ptr<const IDictionaryBase> getDictionary(const String & dictionary_name)
{
auto dict = std::atomic_load(&dictionary);
if (dict)
return dict;
String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name);
dict = external_loader.getDictionary(resolved_name);
context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName());
std::atomic_store(&dictionary, dict);
auto dict = external_loader.getDictionary(resolved_name);
if (!access_checked)
{
context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName());
access_checked = true;
}
return dict;
}
@ -122,6 +122,8 @@ private:
const Context & context;
const ExternalDictionariesLoader & external_loader;
mutable std::shared_ptr<const IDictionaryBase> dictionary;
/// Access cannot be not granted, since in this case checkAccess() will throw and access_checked will not be updated.
std::atomic<bool> access_checked = false;
};

View File

@ -290,38 +290,38 @@ private:
/// Apply target function by feeding it "batches" of N columns
/// Combining 10 columns per pass is the fastest for large columns sizes.
/// For small columns sizes - more columns is faster.
/// Combining 8 columns per pass is the fastest method, because it's the maximum when clang vectorizes a loop.
template <
typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 10>
typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 8>
struct OperationApplier
{
template <typename Columns, typename ResultData>
static void apply(Columns & in, ResultData & result_data, bool use_result_data_as_input = false)
{
if (!use_result_data_as_input)
doBatchedApply<false>(in, result_data);
doBatchedApply<false>(in, result_data.data(), result_data.size());
while (!in.empty())
doBatchedApply<true>(in, result_data);
doBatchedApply<true>(in, result_data.data(), result_data.size());
}
template <bool CarryResult, typename Columns, typename ResultData>
static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data)
template <bool CarryResult, typename Columns, typename Result>
static void NO_INLINE doBatchedApply(Columns & in, Result * __restrict result_data, size_t size)
{
if (N > in.size())
{
OperationApplier<Op, OperationApplierImpl, N - 1>
::template doBatchedApply<CarryResult>(in, result_data);
::template doBatchedApply<CarryResult>(in, result_data, size);
return;
}
const OperationApplierImpl<Op, N> operation_applier_impl(in);
size_t i = 0;
for (auto & res : result_data)
for (size_t i = 0; i < size; ++i)
{
if constexpr (CarryResult)
res = Op::apply(res, operation_applier_impl.apply(i++));
result_data[i] = Op::apply(result_data[i], operation_applier_impl.apply(i));
else
res = operation_applier_impl.apply(i++);
result_data[i] = operation_applier_impl.apply(i);
}
in.erase(in.end() - N, in.end());
}
@ -332,7 +332,7 @@ template <
struct OperationApplier<Op, OperationApplierImpl, 0>
{
template <bool, typename Columns, typename Result>
static void NO_INLINE doBatchedApply(Columns &, Result &)
static void NO_INLINE doBatchedApply(Columns &, Result &, size_t)
{
throw Exception(
"OperationApplier<...>::apply(...): not enough arguments to run this method",

View File

@ -159,7 +159,7 @@ struct NumIfImpl<A, B, NumberTraits::Error>
private:
[[noreturn]] static void throwError()
{
throw Exception("Invalid types of arguments 2 and 3 of if", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
throw Exception("Incompatible types of arguments corresponding to two conditional branches", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
public:
template <typename... Args> static ColumnPtr vectorVector(Args &&...) { throwError(); }

View File

@ -582,7 +582,7 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr & ast,
/// Special check for WITH statement alias. Add alias action to be able to use this alias.
if (identifier.prefer_alias_to_column_name && !identifier.alias.empty())
data.addAlias(identifier.name, identifier.alias);
data.addAlias(identifier.name(), identifier.alias);
}
}

View File

@ -24,11 +24,12 @@ namespace DB
class AddDefaultDatabaseVisitor
{
public:
AddDefaultDatabaseVisitor(const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr)
: database_name(database_name_),
only_replace_current_database_function(only_replace_current_database_function_),
visit_depth(0),
ostr(ostr_)
explicit AddDefaultDatabaseVisitor(
const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr)
: database_name(database_name_)
, only_replace_current_database_function(only_replace_current_database_function_)
, visit_depth(0)
, ostr(ostr_)
{}
void visitDDL(ASTPtr & ast) const
@ -105,7 +106,7 @@ private:
void visit(const ASTIdentifier & identifier, ASTPtr & ast) const
{
if (!identifier.compound())
ast = createTableIdentifier(database_name, identifier.name);
ast = createTableIdentifier(database_name, identifier.name());
}
void visit(ASTSubquery & subquery, ASTPtr &) const
@ -116,7 +117,7 @@ private:
void visit(ASTFunction & function, ASTPtr &) const
{
bool is_operator_in = false;
for (auto name : {"in", "notIn", "globalIn", "globalNotIn"})
for (const auto * name : {"in", "notIn", "globalIn", "globalNotIn"})
{
if (function.name == name)
{

View File

@ -98,33 +98,33 @@ private:
if (!IdentifierSemantic::getColumnName(node))
return;
auto split = Nested::splitName(node.name); /// ParsedParams, Key1
auto split = Nested::splitName(node.name()); /// ParsedParams, Key1
if (array_join_alias_to_name.count(node.name))
if (array_join_alias_to_name.count(node.name()))
{
/// ARRAY JOIN was written with an array column. Example: SELECT K1 FROM ... ARRAY JOIN ParsedParams.Key1 AS K1
array_join_result_to_source[node.name] = array_join_alias_to_name[node.name]; /// K1 -> ParsedParams.Key1
array_join_result_to_source[node.name()] = array_join_alias_to_name[node.name()]; /// K1 -> ParsedParams.Key1
}
else if (array_join_alias_to_name.count(split.first) && !split.second.empty())
{
/// ARRAY JOIN was written with a nested table. Example: SELECT PP.KEY1 FROM ... ARRAY JOIN ParsedParams AS PP
array_join_result_to_source[node.name] /// PP.Key1 -> ParsedParams.Key1
array_join_result_to_source[node.name()] /// PP.Key1 -> ParsedParams.Key1
= Nested::concatenateName(array_join_alias_to_name[split.first], split.second);
}
else if (array_join_name_to_alias.count(node.name))
else if (array_join_name_to_alias.count(node.name()))
{
/** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams.Key1 AS PP.Key1.
* That is, the query uses the original array, replicated by itself.
*/
array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1
array_join_name_to_alias[node.name]] = node.name;
array_join_name_to_alias[node.name()]] = node.name();
}
else if (array_join_name_to_alias.count(split.first) && !split.second.empty())
{
/** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams AS PP.
*/
array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1
Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name;
Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name();
}
}
};

View File

@ -614,13 +614,18 @@ const std::string & Cluster::ShardInfo::pathForInsert(bool prefer_localhost_repl
if (!has_internal_replication)
throw Exception("internal_replication is not set", ErrorCodes::LOGICAL_ERROR);
if (dir_name_for_internal_replication.empty() || dir_name_for_internal_replication_with_local.empty())
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
if (prefer_localhost_replica)
{
if (dir_name_for_internal_replication.empty())
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
return dir_name_for_internal_replication;
}
else
{
if (dir_name_for_internal_replication_with_local.empty())
throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR);
return dir_name_for_internal_replication_with_local;
}
}
bool Cluster::maybeCrossReplication() const

View File

@ -144,11 +144,11 @@ std::pair<size_t, size_t> CollectJoinOnKeysMatcher::getTableNumbers(const ASTPtr
const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifier * identifier, const Aliases & aliases)
{
if (identifier->compound())
if (identifier->supposedToBeCompound())
return identifier;
UInt32 max_attempts = 100;
for (auto it = aliases.find(identifier->name); it != aliases.end();)
for (auto it = aliases.find(identifier->name()); it != aliases.end();)
{
const ASTIdentifier * parent = identifier;
identifier = it->second->as<ASTIdentifier>();
@ -156,12 +156,12 @@ const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifie
break; /// not a column alias
if (identifier == parent)
break; /// alias to itself with the same name: 'a as a'
if (identifier->compound())
if (identifier->supposedToBeCompound())
break; /// not an alias. Break to prevent cycle through short names: 'a as b, t1.b as a'
it = aliases.find(identifier->name);
it = aliases.find(identifier->name());
if (!max_attempts--)
throw Exception("Cannot unroll aliases for '" + identifier->name + "'", ErrorCodes::LOGICAL_ERROR);
throw Exception("Cannot unroll aliases for '" + identifier->name() + "'", ErrorCodes::LOGICAL_ERROR);
}
return identifier;
@ -186,7 +186,7 @@ size_t CollectJoinOnKeysMatcher::getTableForIdentifiers(std::vector<const ASTIde
if (!membership)
{
const String & name = identifier->name;
const String & name = identifier->name();
bool in_left_table = data.left_table.hasColumn(name);
bool in_right_table = data.right_table.hasColumn(name);

View File

@ -335,7 +335,6 @@ struct ContextShared
std::optional<BackgroundProcessingPool> background_move_pool; /// The thread pool for the background moves performed by the tables.
std::optional<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables)
std::optional<BackgroundSchedulePool> distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends)
std::optional<BackgroundSchedulePool> message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used in kafka streaming)
MultiVersion<Macros> macros; /// Substitutions extracted from config.
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
/// Rules for selecting the compression settings, depending on the size of the part.
@ -438,7 +437,6 @@ struct ContextShared
schedule_pool.reset();
distributed_schedule_pool.reset();
ddl_worker.reset();
message_broker_schedule_pool.reset();
/// Stop trace collector if any
trace_collector.reset();
@ -1441,17 +1439,6 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool()
return *shared->distributed_schedule_pool;
}
BackgroundSchedulePool & Context::getMessageBrokerSchedulePool()
{
auto lock = getLock();
if (!shared->message_broker_schedule_pool)
shared->message_broker_schedule_pool.emplace(
settings.background_message_broker_schedule_pool_size,
CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask,
"BgMBSchPool");
return *shared->message_broker_schedule_pool;
}
void Context::setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker)
{
auto lock = getLock();

View File

@ -511,7 +511,6 @@ public:
BackgroundProcessingPool & getBackgroundPool();
BackgroundProcessingPool & getBackgroundMovePool();
BackgroundSchedulePool & getSchedulePool();
BackgroundSchedulePool & getMessageBrokerSchedulePool();
BackgroundSchedulePool & getDistributedSchedulePool();
void setDDLWorker(std::unique_ptr<DDLWorker> ddl_worker);

View File

@ -532,7 +532,7 @@ std::unique_ptr<DDLGuard> DatabaseCatalog::getDDLGuard(const String & database,
std::unique_lock lock(ddl_guards_mutex);
auto db_guard_iter = ddl_guards.try_emplace(database).first;
DatabaseGuard & db_guard = db_guard_iter->second;
return std::make_unique<DDLGuard>(db_guard.first, db_guard.second, std::move(lock), table);
return std::make_unique<DDLGuard>(db_guard.first, db_guard.second, std::move(lock), table, database);
}
std::unique_lock<std::shared_mutex> DatabaseCatalog::getExclusiveDDLGuardForDatabase(const String & database)
@ -834,7 +834,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
}
DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem)
DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem, const String & database_name)
: map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_))
{
it = map.emplace(elem, Entry{std::make_unique<std::mutex>(), 0}).first;
@ -843,14 +843,19 @@ DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<s
table_lock = std::unique_lock(*it->second.mutex);
bool is_database = elem.empty();
if (!is_database)
db_mutex.lock_shared();
{
bool locked_database_for_read = db_mutex.try_lock_shared();
if (!locked_database_for_read)
{
removeTableLock();
throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} is currently dropped or renamed", database_name);
}
}
}
DDLGuard::~DDLGuard()
void DDLGuard::removeTableLock()
{
bool is_database = it->first.empty();
if (!is_database)
db_mutex.unlock_shared();
guards_lock.lock();
--it->second.counter;
if (!it->second.counter)
@ -860,4 +865,12 @@ DDLGuard::~DDLGuard()
}
}
DDLGuard::~DDLGuard()
{
bool is_database = it->first.empty();
if (!is_database)
db_mutex.unlock_shared();
removeTableLock();
}
}

View File

@ -51,7 +51,7 @@ public:
/// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion.
using Map = std::map<String, Entry>;
DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem);
DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock<std::mutex> guards_lock_, const String & elem, const String & database_name);
~DDLGuard();
private:
@ -60,6 +60,8 @@ private:
Map::iterator it;
std::unique_lock<std::mutex> guards_lock;
std::unique_lock<std::mutex> table_lock;
void removeTableLock();
};

View File

@ -316,6 +316,7 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
{
auto & result = sample_block.getByName(result_name);
result.type = result_type;
result.name = result_name;
result.column = source.column;
}
else
@ -1560,6 +1561,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction(
ColumnWithTypeAndName argument;
argument.column = child.column;
argument.type = child.result_type;
argument.name = child.result_name;
if (!argument.column || !isColumnConst(*argument.column))
all_const = false;

View File

@ -41,7 +41,7 @@ void ExpressionInfoMatcher::visit(const ASTIdentifier & identifier, const ASTPtr
const auto & table = data.tables[index];
// TODO: make sure no collision ever happens
if (table.hasColumn(identifier.name))
if (table.hasColumn(identifier.name()))
{
data.unique_reference_tables_pos.emplace(index);
break;

View File

@ -51,7 +51,7 @@ std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std
if ((best_match != ColumnMatch::NoMatch) && same_match)
{
if (!allow_ambiguous)
throw Exception("Ambiguous column '" + identifier.name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
throw Exception("Ambiguous column '" + identifier.name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
best_match = ColumnMatch::Ambiguous;
return {};
}
@ -66,7 +66,7 @@ std::optional<size_t> tryChooseTable(const ASTIdentifier & identifier, const std
std::optional<String> IdentifierSemantic::getColumnName(const ASTIdentifier & node)
{
if (!node.semantic->special)
return node.name;
return node.name();
return {};
}
@ -75,14 +75,14 @@ std::optional<String> IdentifierSemantic::getColumnName(const ASTPtr & ast)
if (ast)
if (const auto * id = ast->as<ASTIdentifier>())
if (!id->semantic->special)
return id->name;
return id->name();
return {};
}
std::optional<String> IdentifierSemantic::getTableName(const ASTIdentifier & node)
{
if (node.semantic->special)
return node.name;
return node.name();
return {};
}
@ -91,7 +91,7 @@ std::optional<String> IdentifierSemantic::getTableName(const ASTPtr & ast)
if (ast)
if (const auto * id = ast->as<ASTIdentifier>())
if (id->semantic->special)
return id->name;
return id->name();
return {};
}
@ -151,7 +151,7 @@ StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & iden
if (identifier.name_parts.size() == 2)
return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid };
return { "", identifier.name, identifier.uuid };
return { "", identifier.name_parts[0], identifier.uuid };
}
std::optional<String> IdentifierSemantic::extractNestedName(const ASTIdentifier & identifier, const String & table_name)
@ -232,16 +232,8 @@ void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const Da
if (!to_strip)
return;
std::vector<String> stripped(identifier.name_parts.begin() + to_strip, identifier.name_parts.end());
DB::String new_name;
for (const auto & part : stripped)
{
if (!new_name.empty())
new_name += '.';
new_name += part;
}
identifier.name.swap(new_name);
identifier.name_parts = std::vector<String>(identifier.name_parts.begin() + to_strip, identifier.name_parts.end());
identifier.resetFullName();
}
void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table)
@ -249,10 +241,11 @@ void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const Dat
String prefix = db_and_table.getQualifiedNamePrefix();
if (!prefix.empty())
{
String short_name = identifier.shortName();
identifier.name = prefix + short_name;
prefix.resize(prefix.size() - 1); /// crop dot
identifier.name_parts = {prefix, short_name};
identifier.name_parts = {prefix, identifier.shortName()};
identifier.resetFullName();
identifier.semantic->table = prefix;
identifier.semantic->legacy_compound = true;
}
}

View File

@ -10,10 +10,12 @@ namespace DB
struct IdentifierSemanticImpl
{
bool special = false; /// for now it's 'not a column': tables, subselects and some special stuff like FORMAT
bool can_be_alias = true; /// if it's a cropped name it could not be an alias
bool covered = false; /// real (compound) name is hidden by an alias (short name)
std::optional<size_t> membership; /// table position in join
bool special = false; /// for now it's 'not a column': tables, subselects and some special stuff like FORMAT
bool can_be_alias = true; /// if it's a cropped name it could not be an alias
bool covered = false; /// real (compound) name is hidden by an alias (short name)
std::optional<size_t> membership; /// table position in join
String table = {}; /// store table name for columns just to support legacy logic.
bool legacy_compound = false; /// true if identifier supposed to be comply for legacy |compound()| behavior
};
/// Static class to manipulate IdentifierSemanticImpl via ASTIdentifier

View File

@ -29,7 +29,15 @@ public:
if constexpr (!_top_to_bottom)
visitChildren(ast);
Matcher::visit(ast, data);
try
{
Matcher::visit(ast, data);
}
catch (Exception & e)
{
e.addMessage("While processing {}", ast->formatForErrorMessage());
throw;
}
if constexpr (_top_to_bottom)
visitChildren(ast);

View File

@ -47,40 +47,37 @@ BlockIO InterpreterDropQuery::execute()
if (!drop.table.empty())
{
if (!drop.is_dictionary)
return executeToTable({drop.database, drop.table, drop.uuid}, drop);
return executeToTable(drop);
else
return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock);
}
else if (!drop.database.empty())
return executeToDatabase(drop.database, drop.kind, drop.if_exists);
return executeToDatabase(drop.database, drop.kind, drop.if_exists, drop.no_delay);
else
throw Exception("Nothing to drop, both names are empty", ErrorCodes::LOGICAL_ERROR);
}
BlockIO InterpreterDropQuery::executeToTable(
const StorageID & table_id_,
const ASTDropQuery & query)
BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query)
{
if (query.temporary || table_id_.database_name.empty())
/// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard
auto table_id = StorageID(query);
if (query.temporary || table_id.database_name.empty())
{
if (context.tryResolveStorageID(table_id_, Context::ResolveExternal))
return executeToTemporaryTable(table_id_.getTableName(), query.kind);
if (context.tryResolveStorageID(table_id, Context::ResolveExternal))
return executeToTemporaryTable(table_id.getTableName(), query.kind);
else
table_id.database_name = context.getCurrentDatabase();
}
if (query.temporary)
{
if (query.if_exists)
return {};
throw Exception("Temporary table " + backQuoteIfNeed(table_id_.table_name) + " doesn't exist",
throw Exception("Temporary table " + backQuoteIfNeed(table_id.table_name) + " doesn't exist",
ErrorCodes::UNKNOWN_TABLE);
}
auto table_id = query.if_exists ? context.tryResolveStorageID(table_id_, Context::ResolveOrdinary)
: context.resolveStorageID(table_id_, Context::ResolveOrdinary);
if (!table_id)
return {};
auto ddl_guard = (!query.no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name) : nullptr);
/// If table was already dropped by anyone, an exception will be thrown
@ -92,6 +89,9 @@ BlockIO InterpreterDropQuery::executeToTable(
if (query_ptr->as<ASTDropQuery &>().is_view && !table->isView())
throw Exception("Table " + table_id.getNameForLogs() + " is not a View", ErrorCodes::LOGICAL_ERROR);
/// Now get UUID, so we can wait for table data to be finally dropped
table_id.uuid = database->tryGetTableUUID(table_id.table_name);
if (query.kind == ASTDropQuery::Kind::Detach)
{
context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id);
@ -223,7 +223,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name,
}
BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists)
BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay)
{
auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
@ -251,11 +251,16 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS
ASTDropQuery query;
query.kind = kind;
query.if_exists = true;
query.database = database_name;
query.no_delay = no_delay;
for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
{
/// Reset reference counter of the StoragePtr to allow synchronous drop.
iterator->reset();
query.table = iterator->name();
executeToTable({query.database, query.table}, query);
executeToTable(query);
}
}

View File

@ -29,9 +29,9 @@ private:
ASTPtr query_ptr;
Context & context;
BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists);
BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay);
BlockIO executeToTable(const StorageID & table_id, const ASTDropQuery & query);
BlockIO executeToTable(const ASTDropQuery & query);
BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock);

View File

@ -117,7 +117,7 @@ private:
throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR);
ASTIdentifier & identifier = child->children[0]->as<ASTIdentifier &>();
data.addTableColumns(identifier.name);
data.addTableColumns(identifier.name());
}
else
data.new_select_expression_list->children.push_back(child);
@ -228,7 +228,7 @@ struct CollectColumnIdentifiersMatcher
void addIdentirier(const ASTIdentifier & ident)
{
for (const auto & aliases : ignored)
if (aliases.count(ident.name))
if (aliases.count(ident.name()))
return;
identifiers.push_back(const_cast<ASTIdentifier *>(&ident));
}
@ -293,7 +293,7 @@ struct CheckAliasDependencyVisitorData
void visit(ASTIdentifier & ident, ASTPtr &)
{
if (!dependency && aliases.count(ident.name))
if (!dependency && aliases.count(ident.name()))
dependency = &ident;
}
};
@ -467,7 +467,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
for (ASTIdentifier * ident : identifiers)
{
bool got_alias = aliases.count(ident->name);
bool got_alias = aliases.count(ident->name());
bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias
if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous))
@ -475,12 +475,12 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
if (!ident->isShort())
{
if (got_alias)
throw Exception("Alias clashes with qualified column '" + ident->name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
throw Exception("Alias clashes with qualified column '" + ident->name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME);
String short_name = ident->shortName();
String original_long_name;
if (public_identifiers.count(ident))
original_long_name = ident->name;
original_long_name = ident->name();
size_t count = countTablesWithColumn(tables, short_name);
@ -488,7 +488,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
{
const auto & table = tables[*table_pos];
IdentifierSemantic::setColumnLongName(*ident, table.table); /// table.column -> table_alias.column
auto & unique_long_name = ident->name;
const auto & unique_long_name = ident->name();
/// For tables moved into subselects we need unique short names for clashed names
if (*table_pos != last_table_pos)
@ -512,7 +512,7 @@ std::vector<TableNeededColumns> normalizeColumnNamesExtractNeeded(
needed_columns[*table_pos].no_clashes.emplace(ident->shortName());
}
else if (!got_alias)
throw Exception("Unknown column name '" + ident->name + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
throw Exception("Unknown column name '" + ident->name() + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
}
return needed_columns;
@ -613,12 +613,12 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast
{
for (auto * ident : on_identifiers)
{
auto it = data.aliases.find(ident->name);
if (!on_aliases.count(ident->name) && it != data.aliases.end())
auto it = data.aliases.find(ident->name());
if (!on_aliases.count(ident->name()) && it != data.aliases.end())
{
auto alias_expression = it->second;
alias_pushdown[table_pos].push_back(alias_expression);
on_aliases[ident->name] = alias_expression;
on_aliases[ident->name()] = alias_expression;
}
}
}
@ -638,14 +638,14 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast
CheckAliasDependencyVisitor(check).visit(expr.second);
if (check.dependency)
throw Exception("Cannot rewrite JOINs. Alias '" + expr.first +
"' used in ON section depends on another alias '" + check.dependency->name + "'",
"' used in ON section depends on another alias '" + check.dependency->name() + "'",
ErrorCodes::NOT_IMPLEMENTED);
}
/// Check same name in aliases, USING and ON sections. Cannot push down alias to ON through USING cause of name masquerading.
for (auto * ident : using_identifiers)
if (on_aliases.count(ident->name))
throw Exception("Cannot rewrite JOINs. Alias '" + ident->name + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED);
if (on_aliases.count(ident->name()))
throw Exception("Cannot rewrite JOINs. Alias '" + ident->name() + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED);
using_identifiers.clear();
/// Replace pushdowned expressions with aliases names in original expression lists.

View File

@ -49,7 +49,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query)
if (table_expr.database_and_table_name)
{
const auto & table_id = table_expr.database_and_table_name->as<ASTIdentifier &>();
String expr = "(select * from " + table_id.name + ") as " + table_id.shortName();
String expr = "(select * from " + table_id.name() + ") as " + table_id.shortName();
// FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b",
// we can't replace aliased tables.
@ -99,7 +99,7 @@ private:
match == IdentifierSemantic::ColumnMatch::DbAndTable)
{
if (rewritten)
throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name + "'",
throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name() + "'",
ErrorCodes::AMBIGUOUS_COLUMN_NAME);
/// Table has an alias. So we set a new name qualified by table alias.
IdentifierSemantic::setColumnLongName(identifier, table);
@ -114,10 +114,10 @@ private:
bool rewritten = false;
for (const auto & table : data)
{
if (identifier.name == table.table)
if (identifier.name() == table.table)
{
if (rewritten)
throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name + "'",
throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name() + "'",
ErrorCodes::AMBIGUOUS_COLUMN_NAME);
identifier.setShortName(table.alias);
rewritten = true;

View File

@ -73,8 +73,8 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
return;
/// If it is an alias, but not a parent alias (for constructs like "SELECT column + 1 AS column").
auto it_alias = data.aliases.find(node.name);
if (it_alias != data.aliases.end() && current_alias != node.name)
auto it_alias = data.aliases.find(node.name());
if (it_alias != data.aliases.end() && current_alias != node.name())
{
if (!IdentifierSemantic::canBeAlias(node))
return;
@ -89,7 +89,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
String node_alias = ast->tryGetAlias();
if (current_asts.count(alias_node.get()) /// We have loop of multiple aliases
|| (node.name == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop
|| (node.name() == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop
throw Exception("Cyclic aliases", ErrorCodes::CYCLIC_ALIASES);
/// Let's replace it with the corresponding tree node.
@ -97,7 +97,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
{
/// Avoid infinite recursion here
auto opt_name = IdentifierSemantic::getColumnName(alias_node);
bool is_cycle = opt_name && *opt_name == node.name;
bool is_cycle = opt_name && *opt_name == node.name();
if (!is_cycle)
{

View File

@ -3,10 +3,13 @@
namespace DB
{
void RenameColumnData::visit(ASTIdentifier & identifier, ASTPtr &) const
{
// TODO(ilezhankin): make proper rename
std::optional<String> identifier_column_name = IdentifierSemantic::getColumnName(identifier);
if (identifier_column_name && identifier_column_name == column_name)
identifier.name = rename_to;
identifier.setShortName(rename_to);
}
}

View File

@ -27,7 +27,7 @@ void RequiredSourceColumnsData::addColumnIdentifier(const ASTIdentifier & node)
/// There should be no complex cases after query normalization. Names to aliases: one-to-many.
String alias = node.tryGetAlias();
required_names[node.name].addInclusion(alias);
required_names[node.name()].addInclusion(alias);
}
bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast)
@ -42,7 +42,7 @@ bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast)
void RequiredSourceColumnsData::addArrayJoinIdentifier(const ASTIdentifier & node)
{
array_join_columns.insert(node.name);
array_join_columns.insert(node.name());
}
size_t RequiredSourceColumnsData::nameInclusion(const String & name) const

View File

@ -34,7 +34,7 @@ std::vector<String> RequiredSourceColumnsMatcher::extractNamesFromLambda(const A
if (!identifier)
throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH);
names.push_back(identifier->name);
names.push_back(identifier->name());
}
return names;
@ -132,10 +132,11 @@ void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const AS
void RequiredSourceColumnsMatcher::visit(const ASTIdentifier & node, const ASTPtr &, Data & data)
{
if (node.name.empty())
// FIXME(ilezhankin): shouldn't ever encounter
if (node.name().empty())
throw Exception("Expected not empty name", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!data.private_aliases.count(node.name))
if (!data.private_aliases.count(node.name()))
data.addColumnIdentifier(node);
}

View File

@ -300,8 +300,8 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
performance_counters.setParent(&ProfileEvents::global_counters);
memory_tracker.reset();
/// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below.
memory_tracker.setParent(nullptr);
/// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below (will reset to its parent).
memory_tracker.setParent(thread_group->memory_tracker.getParent());
query_id.clear();
query_context = nullptr;

View File

@ -104,7 +104,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &,
if (data.unknownColumn(table_pos, identifier))
{
String table_name = data.tables[table_pos].table.getQualifiedNamePrefix(false);
throw Exception("There's no column '" + identifier.name + "' in table '" + table_name + "'",
throw Exception("There's no column '" + identifier.name() + "' in table '" + table_name + "'",
ErrorCodes::UNKNOWN_IDENTIFIER);
}
@ -175,9 +175,12 @@ void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr
static void addIdentifier(ASTs & nodes, const DatabaseAndTableWithAlias & table, const String & column_name)
{
std::vector<String> parts = {column_name};
String table_name = table.getQualifiedNamePrefix(false);
auto identifier = std::make_shared<ASTIdentifier>(std::vector<String>{table_name, column_name});
nodes.emplace_back(identifier);
if (!table_name.empty()) parts.insert(parts.begin(), table_name);
nodes.emplace_back(std::make_shared<ASTIdentifier>(std::move(parts)));
}
/// Replace *, alias.*, database.table.* with a list of columns.
@ -354,7 +357,7 @@ void RestoreQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, D
{
if (IdentifierSemantic::getMembership(identifier))
{
identifier.restoreCompoundName();
identifier.restoreTable(); // TODO(ilezhankin): should restore qualified name here - why exactly here?
if (data.rename)
data.changeTable(identifier);
}

View File

@ -72,7 +72,7 @@ ASTPtr evaluateConstantExpressionAsLiteral(const ASTPtr & node, const Context &
ASTPtr evaluateConstantExpressionOrIdentifierAsLiteral(const ASTPtr & node, const Context & context)
{
if (const auto * id = node->as<ASTIdentifier>())
return std::make_shared<ASTLiteral>(id->name);
return std::make_shared<ASTLiteral>(id->name());
return evaluateConstantExpressionAsLiteral(node, context);
}
@ -113,7 +113,7 @@ namespace
const auto & name = name_and_type.name;
const auto & type = name_and_type.type;
if (name == identifier->name)
if (name == identifier->name())
{
ColumnWithTypeAndName column;
Field converted = convertFieldToType(value, *type);

View File

@ -19,7 +19,7 @@ namespace ErrorCodes
std::string getClusterName(const IAST & node)
{
if (const auto * ast_id = node.as<ASTIdentifier>())
return ast_id->name;
return ast_id->name();
if (const auto * ast_lit = node.as<ASTLiteral>())
return ast_lit->value.safeGet<String>();

View File

@ -128,7 +128,7 @@ int main(int argc, char ** argv)
std::cerr << "sum_counts: " << sum_counts << ", elems: " << elems << std::endl;
if (sum_counts != n)
std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
std::cerr << "Error!" << std::endl;
}
return 0;

View File

@ -33,14 +33,32 @@ void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes)
void ASTColumnsApplyTransformer::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "APPLY" << (settings.hilite ? hilite_none : "") << " " << func_name;
settings.ostr << (settings.hilite ? hilite_keyword : "") << "APPLY" << (settings.hilite ? hilite_none : "") << " ";
if (!column_name_prefix.empty())
settings.ostr << "(" << func_name << ", '" << column_name_prefix << "')";
else
settings.ostr << func_name;
}
void ASTColumnsApplyTransformer::transform(ASTs & nodes) const
{
for (auto & column : nodes)
{
String name;
auto alias = column->tryGetAlias();
if (!alias.empty())
name = alias;
else
{
if (const auto * id = column->as<ASTIdentifier>())
name = id->shortName();
else
name = column->getColumnName();
}
column = makeASTFunction(func_name, column);
if (!column_name_prefix.empty())
column->setAlias(column_name_prefix + name);
}
}
@ -78,7 +96,7 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
{
for (int i = children.size() - 1; i >= 0; --i)
{
if (children[i]->as<const ASTIdentifier &>().name == id->shortName())
if (children[i]->as<const ASTIdentifier &>().name() == id->shortName())
{
expected_columns.erase(expected_columns.begin() + i);
return true;
@ -96,7 +114,7 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
{
if (i > 0)
expected_columns_str += ", ";
expected_columns_str += expected_columns[i]->as<const ASTIdentifier &>().name;
expected_columns_str += expected_columns[i]->as<const ASTIdentifier &>().name();
}
throw Exception(

View File

@ -22,6 +22,7 @@ public:
}
void transform(ASTs & nodes) const override;
String func_name;
String column_name_prefix;
protected:
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;

View File

@ -1,10 +1,10 @@
#include <Common/typeid_cast.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/queryToString.h>
#include <IO/WriteBufferFromOStream.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/IdentifierSemantic.h>
#include <Interpreters/StorageID.h>
#include <Parsers/queryToString.h>
namespace DB
@ -16,6 +16,27 @@ namespace ErrorCodes
extern const int SYNTAX_ERROR;
}
ASTIdentifier::ASTIdentifier(const String & short_name)
: full_name(short_name), name_parts{short_name}, semantic(std::make_shared<IdentifierSemanticImpl>())
{
assert(!full_name.empty());
}
ASTIdentifier::ASTIdentifier(std::vector<String> && name_parts_, bool special)
: name_parts(name_parts_), semantic(std::make_shared<IdentifierSemanticImpl>())
{
assert(!name_parts.empty());
for (const auto & part [[maybe_unused]] : name_parts)
assert(!part.empty());
semantic->special = special;
semantic->legacy_compound = true;
if (!special && name_parts.size() >= 2)
semantic->table = name_parts.end()[-2];
resetFullName();
}
ASTPtr ASTIdentifier::clone() const
{
@ -24,51 +45,29 @@ ASTPtr ASTIdentifier::clone() const
return ret;
}
std::shared_ptr<ASTIdentifier> ASTIdentifier::createSpecial(const String & name, std::vector<String> && name_parts)
bool ASTIdentifier::supposedToBeCompound() const
{
auto ret = std::make_shared<ASTIdentifier>(name, std::move(name_parts));
ret->semantic->special = true;
return ret;
return semantic->legacy_compound;
}
ASTIdentifier::ASTIdentifier(const String & name_, std::vector<String> && name_parts_)
: name(name_)
, name_parts(name_parts_)
, semantic(std::make_shared<IdentifierSemanticImpl>())
{
if (!name_parts.empty() && name_parts[0].empty())
name_parts.erase(name_parts.begin());
if (name.empty())
{
if (name_parts.size() == 2)
name = name_parts[0] + '.' + name_parts[1];
else if (name_parts.size() == 1)
name = name_parts[0];
}
}
ASTIdentifier::ASTIdentifier(std::vector<String> && name_parts_)
: ASTIdentifier("", std::move(name_parts_))
{}
void ASTIdentifier::setShortName(const String & new_name)
{
name = new_name;
name_parts.clear();
assert(!new_name.empty());
full_name = new_name;
name_parts = {new_name};
bool special = semantic->special;
*semantic = IdentifierSemanticImpl();
semantic->special = special;
}
void ASTIdentifier::restoreCompoundName()
const String & ASTIdentifier::name() const
{
if (name_parts.empty())
return;
name = name_parts[0];
for (size_t i = 1; i < name_parts.size(); ++i)
name += '.' + name_parts[i];
assert(!name_parts.empty());
assert(!full_name.empty());
return full_name;
}
void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const
@ -93,20 +92,29 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form
}
else
{
format_element(name);
format_element(shortName());
}
}
void ASTIdentifier::appendColumnNameImpl(WriteBuffer & ostr) const
{
writeString(name, ostr);
writeString(name(), ostr);
}
void ASTIdentifier::restoreTable()
{
if (!compound())
{
name_parts.insert(name_parts.begin(), semantic->table);
resetFullName();
}
}
void ASTIdentifier::resetTable(const String & database_name, const String & table_name)
{
auto ast = createTableIdentifier(database_name, table_name);
auto & ident = ast->as<ASTIdentifier &>();
name.swap(ident.name);
full_name.swap(ident.full_name);
name_parts.swap(ident.name_parts);
uuid = ident.uuid;
}
@ -117,6 +125,13 @@ void ASTIdentifier::updateTreeHashImpl(SipHash & hash_state) const
IAST::updateTreeHashImpl(hash_state);
}
void ASTIdentifier::resetFullName()
{
full_name = name_parts[0];
for (size_t i = 1; i < name_parts.size(); ++i)
full_name += '.' + name_parts[i];
}
ASTPtr createTableIdentifier(const String & database_name, const String & table_name)
{
assert(database_name != "_temporary_and_external_tables");
@ -127,9 +142,9 @@ ASTPtr createTableIdentifier(const StorageID & table_id)
{
std::shared_ptr<ASTIdentifier> res;
if (table_id.database_name.empty())
res = ASTIdentifier::createSpecial(table_id.table_name);
res = std::make_shared<ASTIdentifier>(std::vector<String>{table_id.table_name}, true);
else
res = ASTIdentifier::createSpecial(table_id.database_name + "." + table_id.table_name, {table_id.database_name, table_id.table_name});
res = std::make_shared<ASTIdentifier>(std::vector<String>{table_id.database_name, table_id.table_name}, true);
res->uuid = table_id.uuid;
return res;
}
@ -156,7 +171,7 @@ bool tryGetIdentifierNameInto(const IAST * ast, String & name)
{
if (const auto * node = ast->as<ASTIdentifier>())
{
name = node->name;
name = node->name();
return true;
}
}
@ -180,7 +195,7 @@ StorageID getTableIdentifier(const ASTPtr & ast)
if (identifier.name_parts.size() == 2)
return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid };
return { "", identifier.name, identifier.uuid };
return { "", identifier.name_parts[0], identifier.uuid };
}
}

View File

@ -18,59 +18,54 @@ struct StorageID;
class ASTIdentifier : public ASTWithAlias
{
public:
/// The composite identifier will have a concatenated name (of the form a.b.c),
/// and individual components will be available inside the name_parts.
String name;
UUID uuid = UUIDHelpers::Nil;
ASTIdentifier(const String & name_, std::vector<String> && name_parts_ = {});
ASTIdentifier(std::vector<String> && name_parts_);
explicit ASTIdentifier(const String & short_name);
explicit ASTIdentifier(std::vector<String> && name_parts, bool special = false);
/** Get the text that identifies this element. */
String getID(char delim) const override { return "Identifier" + (delim + name); }
String getID(char delim) const override { return "Identifier" + (delim + name()); }
ASTPtr clone() const override;
void collectIdentifierNames(IdentifierNameSet & set) const override
{
set.insert(name);
}
void collectIdentifierNames(IdentifierNameSet & set) const override { set.insert(name()); }
bool compound() const { return !name_parts.empty(); }
bool isShort() const { return name_parts.empty() || name == name_parts.back(); }
bool compound() const { return name_parts.size() > 1; }
bool isShort() const { return name_parts.size() == 1; }
bool supposedToBeCompound() const; // TODO(ilezhankin): get rid of this
void setShortName(const String & new_name);
/// Restore name field from name_parts in case it was cropped by analyzer but we need a full form for future (re)analyze.
void restoreCompoundName();
/// The composite identifier will have a concatenated name (of the form a.b.c),
/// and individual components will be available inside the name_parts.
const String & shortName() const { return name_parts.back(); }
const String & name() const;
const String & shortName() const
{
if (!name_parts.empty())
return name_parts.back();
return name;
}
void restoreTable(); // TODO(ilezhankin): get rid of this
void resetTable(const String & database_name, const String & table_name);
// FIXME: used only when it's needed to rewrite distributed table name to real remote table name.
void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
String full_name;
std::vector<String> name_parts;
void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void appendColumnNameImpl(WriteBuffer & ostr) const override;
private:
using ASTWithAlias::children; /// ASTIdentifier is child free
std::vector<String> name_parts;
std::shared_ptr<IdentifierSemanticImpl> semantic; /// pimpl
static std::shared_ptr<ASTIdentifier> createSpecial(const String & name, std::vector<String> && name_parts = {});
friend struct IdentifierSemantic;
friend ASTPtr createTableIdentifier(const StorageID & table_id);
friend void setIdentifierSpecial(ASTPtr & ast);
friend StorageID getTableIdentifier(const ASTPtr & ast);
void resetFullName();
};

View File

@ -184,16 +184,10 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
.parse(pos, id_list, expected))
return false;
String name;
std::vector<String> parts;
const auto & list = id_list->as<ASTExpressionList &>();
for (const auto & child : list.children)
{
if (!name.empty())
name += '.';
parts.emplace_back(getIdentifierName(child));
name += parts.back();
}
ParserKeyword s_uuid("UUID");
UUID uuid = UUIDHelpers::Nil;
@ -207,9 +201,7 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
uuid = parseFromString<UUID>(ast_uuid->as<ASTLiteral>()->value.get<String>());
}
if (parts.size() == 1)
parts.clear();
node = std::make_shared<ASTIdentifier>(name, std::move(parts));
node = std::make_shared<ASTIdentifier>(std::move(parts));
node->as<ASTIdentifier>()->uuid = uuid;
return true;
@ -789,6 +781,7 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
++pos;
IntervalKind interval_kind;
ASTPtr interval_func_node;
if (parseIntervalKind(pos, expected, interval_kind))
{
/// function(unit, offset, timestamp)
@ -805,6 +798,13 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
if (!ParserExpression().parse(pos, timestamp_node, expected))
return false;
auto interval_expr_list_args = std::make_shared<ASTExpressionList>();
interval_expr_list_args->children = {offset_node};
interval_func_node = std::make_shared<ASTFunction>();
interval_func_node->as<ASTFunction &>().name = interval_kind.toNameOfFunctionToIntervalDataType();
interval_func_node->as<ASTFunction &>().arguments = std::move(interval_expr_list_args);
interval_func_node->as<ASTFunction &>().children.push_back(interval_func_node->as<ASTFunction &>().arguments);
}
else
{
@ -816,27 +816,13 @@ bool ParserDateAddExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
return false;
++pos;
if (!ParserKeyword("INTERVAL").ignore(pos, expected))
return false;
if (!ParserExpression().parse(pos, offset_node, expected))
return false;
if (!parseIntervalKind(pos, expected, interval_kind))
if (!ParserIntervalOperatorExpression{}.parse(pos, interval_func_node, expected))
return false;
}
if (pos->type != TokenType::ClosingRoundBracket)
return false;
++pos;
auto interval_expr_list_args = std::make_shared<ASTExpressionList>();
interval_expr_list_args->children = {offset_node};
auto interval_func_node = std::make_shared<ASTFunction>();
interval_func_node->name = interval_kind.toNameOfFunctionToIntervalDataType();
interval_func_node->arguments = std::move(interval_expr_list_args);
interval_func_node->children.push_back(interval_func_node->arguments);
auto expr_list_args = std::make_shared<ASTExpressionList>();
expr_list_args->children = {timestamp_node, interval_func_node};
@ -1252,6 +1238,19 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
if (!parseIdentifierOrStringLiteral(pos, expected, func_name))
return false;
String column_name_prefix;
if (with_open_round_bracket && pos->type == TokenType::Comma)
{
++pos;
ParserStringLiteral parser_string_literal;
ASTPtr ast_prefix_name;
if (!parser_string_literal.parse(pos, ast_prefix_name, expected))
return false;
column_name_prefix = ast_prefix_name->as<ASTLiteral &>().value.get<const String &>();
}
if (with_open_round_bracket)
{
if (pos->type != TokenType::ClosingRoundBracket)
@ -1261,6 +1260,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
auto res = std::make_shared<ASTColumnsApplyTransformer>();
res->func_name = func_name;
res->column_name_prefix = column_name_prefix;
node = std::move(res);
return true;
}
@ -1687,7 +1687,7 @@ bool ParserFunctionWithKeyValueArguments::parseImpl(Pos & pos, ASTPtr & node, Ex
}
auto function = std::make_shared<ASTFunctionWithKeyValueArguments>(left_bracket_found);
function->name = Poco::toLower(typeid_cast<ASTIdentifier &>(*identifier.get()).name);
function->name = Poco::toLower(identifier->as<ASTIdentifier>()->name());
function->elements = expr_list_args;
function->children.push_back(function->elements);
node = function;

View File

@ -1,13 +1,11 @@
#include <Parsers/IAST.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/parseIntervalKind.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
#include <Common/typeid_cast.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/parseIntervalKind.h>
#include <Common/StringUtils/StringUtils.h>
@ -645,6 +643,47 @@ bool ParserTimestampOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expe
return true;
}
bool ParserIntervalOperatorExpression::parseArgumentAndIntervalKind(
Pos & pos, ASTPtr & expr, IntervalKind & interval_kind, Expected & expected)
{
auto begin = pos;
auto init_expected = expected;
ASTPtr string_literal;
//// A String literal followed INTERVAL keyword,
/// the literal can be a part of an expression or
/// include Number and INTERVAL TYPE at the same time
if (ParserStringLiteral{}.parse(pos, string_literal, expected))
{
String literal;
if (string_literal->as<ASTLiteral &>().value.tryGet(literal))
{
Tokens tokens(literal.data(), literal.data() + literal.size());
Pos token_pos(tokens, 0);
Expected token_expected;
if (!ParserNumber{}.parse(token_pos, expr, token_expected))
return false;
else
{
/// case: INTERVAL '1' HOUR
/// back to begin
if (!token_pos.isValid())
{
pos = begin;
expected = init_expected;
}
else
/// case: INTERVAL '1 HOUR'
return parseIntervalKind(token_pos, token_expected, interval_kind);
}
}
}
// case: INTERVAL expr HOUR
if (!ParserExpressionWithOptionalAlias(false).parse(pos, expr, expected))
return false;
return parseIntervalKind(pos, expected, interval_kind);
}
bool ParserIntervalOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
auto begin = pos;
@ -654,15 +693,8 @@ bool ParserIntervalOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expec
return next_parser.parse(pos, node, expected);
ASTPtr expr;
/// Any expression can be inside, because operator surrounds it.
if (!ParserExpressionWithOptionalAlias(false).parse(pos, expr, expected))
{
pos = begin;
return next_parser.parse(pos, node, expected);
}
IntervalKind interval_kind;
if (!parseIntervalKind(pos, expected, interval_kind))
if (!parseArgumentAndIntervalKind(pos, expr, interval_kind, expected))
{
pos = begin;
return next_parser.parse(pos, node, expected);
@ -716,7 +748,7 @@ bool ParserKeyValuePair::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
}
auto pair = std::make_shared<ASTPair>(with_brackets);
pair->first = Poco::toLower(typeid_cast<ASTIdentifier &>(*identifier.get()).name);
pair->first = Poco::toLower(identifier->as<ASTIdentifier>()->name());
pair->set(pair->second, value);
node = pair;
return true;
@ -729,3 +761,4 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
}
}

View File

@ -5,6 +5,7 @@
#include <Parsers/IParserBase.h>
#include <Parsers/CommonParsers.h>
#include <Common/IntervalKind.h>
namespace DB
{
@ -232,6 +233,9 @@ protected:
const char * getName() const override { return "INTERVAL operator expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
private:
static bool parseArgumentAndIntervalKind(Pos & pos, ASTPtr & expr, IntervalKind & interval_kind, Expected & expected);
};
class ParserAdditiveExpression : public IParserBase

View File

@ -303,9 +303,9 @@ static inline bool parseOtherCommand(IParser::Pos & pos, ASTPtr & node, Expected
OptionDescribe("ENABLE KEYS", "enable_keys", std::make_shared<ParserAlwaysTrue>()),
OptionDescribe("DISABLE KEYS", "enable_keys", std::make_shared<ParserAlwaysFalse>()),
/// TODO: with collate
OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
OptionDescribe("CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared<ParserCharsetName>()),
OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
OptionDescribe("CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared<ParserCharsetOrCollateName>()),
OptionDescribe("LOCK", "lock", std::make_shared<ParserIdentifier>())
}
};

View File

@ -51,8 +51,8 @@ static inline bool parseColumnDeclareOptions(IParser::Pos & pos, ASTPtr & node,
OptionDescribe("UNIQUE", "unique_key", std::make_unique<ParserAlwaysTrue>()),
OptionDescribe("KEY", "primary_key", std::make_unique<ParserAlwaysTrue>()),
OptionDescribe("COMMENT", "comment", std::make_unique<ParserStringLiteral>()),
OptionDescribe("CHARACTER SET", "charset_name", std::make_unique<ParserCharsetName>()),
OptionDescribe("COLLATE", "collate", std::make_unique<ParserCharsetName>()),
OptionDescribe("CHARACTER SET", "charset_name", std::make_unique<ParserCharsetOrCollateName>()),
OptionDescribe("COLLATE", "collate", std::make_unique<ParserCharsetOrCollateName>()),
OptionDescribe("COLUMN_FORMAT", "column_format", std::make_unique<ParserIdentifier>()),
OptionDescribe("STORAGE", "storage", std::make_unique<ParserIdentifier>()),
OptionDescribe("AS", "generated", std::make_unique<ParserExpression>()),

View File

@ -63,7 +63,7 @@ bool ParserDeclareConstraint::parseImpl(IParser::Pos & pos, ASTPtr & node, Expec
declare_constraint->check_expression = index_check_expression;
if (constraint_symbol)
declare_constraint->constraint_name = constraint_symbol->as<ASTIdentifier>()->name;
declare_constraint->constraint_name = constraint_symbol->as<ASTIdentifier>()->name();
node = declare_constraint;
return true;

View File

@ -73,7 +73,7 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_
index_type = "SPATIAL";
if (p_identifier.parse(pos, temp_node, expected))
index_name = temp_node->as<ASTIdentifier>()->name;
index_name = temp_node->as<ASTIdentifier>()->name();
}
else if (ParserKeyword("FULLTEXT").ignore(pos, expected))
{
@ -82,7 +82,7 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_
index_type = "FULLTEXT";
if (p_identifier.parse(pos, temp_node, expected))
index_name = temp_node->as<ASTIdentifier>()->name;
index_name = temp_node->as<ASTIdentifier>()->name();
}
else
{
@ -94,14 +94,14 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_
index_type = "KEY_BTREE"; /// default index type
if (p_identifier.parse(pos, temp_node, expected))
index_name = temp_node->as<ASTIdentifier>()->name;
index_name = temp_node->as<ASTIdentifier>()->name();
if (ParserKeyword("USING").ignore(pos, expected))
{
if (!p_identifier.parse(pos, temp_node, expected))
return false;
index_type = "KEY_" + temp_node->as<ASTIdentifier>()->name;
index_type = "KEY_" + temp_node->as<ASTIdentifier>()->name();
}
}
@ -122,7 +122,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde
if (!p_identifier.parse(pos, temp_node, expected))
return false;
index_name = temp_node->as<ASTIdentifier>()->name;
index_name = temp_node->as<ASTIdentifier>()->name();
}
}
@ -132,7 +132,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde
ParserKeyword("INDEX").ignore(pos, expected);
if (p_identifier.parse(pos, temp_node, expected))
index_name = temp_node->as<ASTIdentifier>()->name; /// reset index_name
index_name = temp_node->as<ASTIdentifier>()->name(); /// reset index_name
index_type = "UNIQUE_BTREE"; /// default btree index_type
if (ParserKeyword("USING").ignore(pos, expected))
@ -140,7 +140,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde
if (!p_identifier.parse(pos, temp_node, expected))
return false;
index_type = "UNIQUE_" + temp_node->as<ASTIdentifier>()->name;
index_type = "UNIQUE_" + temp_node->as<ASTIdentifier>()->name();
}
}
else if (ParserKeyword("PRIMARY KEY").ignore(pos, expected))
@ -151,14 +151,14 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde
if (!p_identifier.parse(pos, temp_node, expected))
return false;
index_type = "PRIMARY_KEY_" + temp_node->as<ASTIdentifier>()->name;
index_type = "PRIMARY_KEY_" + temp_node->as<ASTIdentifier>()->name();
}
}
else if (ParserKeyword("FOREIGN KEY").ignore(pos, expected))
{
index_type = "FOREIGN";
if (p_identifier.parse(pos, temp_node, expected))
index_name = temp_node->as<ASTIdentifier>()->name; /// reset index_name
index_name = temp_node->as<ASTIdentifier>()->name(); /// reset index_name
}
return true;

View File

@ -4,6 +4,7 @@
#include <IO/ReadHelpers.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ExpressionElementParsers.h>
namespace DB
{
@ -94,41 +95,21 @@ bool ParserAlwaysFalse::parseImpl(IParser::Pos & /*pos*/, ASTPtr & node, Expecte
return true;
}
bool ParserCharsetName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &)
bool ParserCharsetOrCollateName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected)
{
/// Identifier in backquotes or in double quotes
if (pos->type == TokenType::QuotedIdentifier)
{
ReadBufferFromMemory buf(pos->begin, pos->size());
String s;
ParserIdentifier p_identifier;
ParserStringLiteral p_string_literal;
if (*pos->begin == '`')
readBackQuotedStringWithSQLStyle(s, buf);
else
readDoubleQuotedStringWithSQLStyle(s, buf);
if (s.empty()) /// Identifiers "empty string" are not allowed.
return false;
node = std::make_shared<ASTIdentifier>(s);
++pos;
if (p_identifier.parse(pos, node, expected))
return true;
}
else if (pos->type == TokenType::BareWord)
else
{
const char * begin = pos->begin;
while (true)
if (p_string_literal.parse(pos, node, expected))
{
if (!isWhitespaceASCII(*pos->end) && pos->type != TokenType::EndOfStream)
++pos;
else
break;
const auto & string_value = node->as<ASTLiteral>()->value.safeGet<String>();
node = std::make_shared<ASTIdentifier>(string_value);
return true;
}
node = std::make_shared<ASTIdentifier>(String(begin, pos->end));
++pos;
return true;
}
return false;

View File

@ -59,12 +59,11 @@ public:
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/// Copy and paste from ParserIdentifier,
/// the difference is that multiple tokens are glued if there is no whitespace ASCII between them
struct ParserCharsetName : public IParserBase
/// identifier, string literal, binary keyword
struct ParserCharsetOrCollateName : public IParserBase
{
protected:
const char * getName() const override { return "charset name"; }
const char * getName() const override { return "charset or collate name"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected &) override;
};

View File

@ -107,7 +107,7 @@ bool ParserDeclarePartition::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect
partition_declare->less_than = less_than;
partition_declare->in_expression = in_expression;
partition_declare->subpartitions = subpartitions;
partition_declare->partition_name = partition_name->as<ASTIdentifier>()->name;
partition_declare->partition_name = partition_name->as<ASTIdentifier>()->name();
if (options)
{

View File

@ -95,7 +95,7 @@ bool ParserDeclareReference::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect
declare_reference->on_delete_option = delete_option;
declare_reference->on_update_option = update_option;
declare_reference->reference_expression = expression;
declare_reference->reference_table_name = table_name->as<ASTIdentifier>()->name;
declare_reference->reference_table_name = table_name->as<ASTIdentifier>()->name();
node = declare_reference;
return true;

Some files were not shown because too many files have changed in this diff Show More