mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
fix conflict
This commit is contained in:
feng lv
commit
fe22563465
@ -184,10 +184,27 @@ endif ()
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
|
||||
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
|
||||
|
||||
if (NOT OBJCOPY_PATH AND OS_DARWIN)
|
||||
find_program (BREW_PATH NAMES "brew")
|
||||
if (BREW_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX)
|
||||
if (LLVM_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
if (NOT OBJCOPY_PATH)
|
||||
execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX)
|
||||
if (BINUTILS_PREFIX)
|
||||
find_program (OBJCOPY_PATH NAMES "objcopy" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH)
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
if (OBJCOPY_PATH)
|
||||
message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
|
||||
message (STATUS "Using objcopy: ${OBJCOPY_PATH}")
|
||||
else ()
|
||||
message(FATAL_ERROR "Cannot find objcopy.")
|
||||
message (FATAL_ERROR "Cannot find objcopy.")
|
||||
endif ()
|
||||
|
||||
if (OS_DARWIN)
|
||||
|
||||
@ -17,7 +17,7 @@ class DateLUT : private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
/// Return singleton DateLUTImpl instance for the default time zone.
|
||||
static ALWAYS_INLINE const DateLUTImpl & instance()
|
||||
static ALWAYS_INLINE const DateLUTImpl & instance() // -V1071
|
||||
{
|
||||
const auto & date_lut = getInstance();
|
||||
return *date_lut.default_impl.load(std::memory_order_acquire);
|
||||
|
||||
@ -119,11 +119,16 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
/// We use Int64 instead of time_t because time_t is mapped to the different types (long or long long)
|
||||
/// on Linux and Darwin (on both of them, long and long long are 64 bit and behaves identically,
|
||||
/// but they are different types in C++ and this affects function overload resolution).
|
||||
using Time = Int64;
|
||||
|
||||
/// The order of fields matters for alignment and sizeof.
|
||||
struct Values
|
||||
{
|
||||
/// time_t at beginning of the day.
|
||||
Int64 date;
|
||||
/// Time at beginning of the day.
|
||||
Time date;
|
||||
|
||||
/// Properties of the day.
|
||||
UInt16 year;
|
||||
@ -182,20 +187,20 @@ private:
|
||||
LUTIndex years_months_lut[DATE_LUT_YEARS * 12];
|
||||
|
||||
/// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time.
|
||||
time_t offset_at_start_of_epoch;
|
||||
Time offset_at_start_of_epoch;
|
||||
/// UTC offset at the beginning of the first supported year.
|
||||
time_t offset_at_start_of_lut;
|
||||
Time offset_at_start_of_lut;
|
||||
bool offset_is_whole_number_of_hours_during_epoch;
|
||||
|
||||
/// Time zone name.
|
||||
std::string time_zone;
|
||||
|
||||
inline LUTIndex findIndex(time_t t) const
|
||||
inline LUTIndex findIndex(Time t) const
|
||||
{
|
||||
/// First guess.
|
||||
Int64 guess = (t / 86400) + daynum_offset_epoch;
|
||||
Time guess = (t / 86400) + daynum_offset_epoch;
|
||||
|
||||
/// For negative time_t the integer division was rounded up, so the guess is offset by one.
|
||||
/// For negative Time the integer division was rounded up, so the guess is offset by one.
|
||||
if (unlikely(t < 0))
|
||||
--guess;
|
||||
|
||||
@ -227,7 +232,7 @@ private:
|
||||
return LUTIndex{static_cast<UInt32>(d + daynum_offset_epoch) & date_lut_mask};
|
||||
}
|
||||
|
||||
inline LUTIndex toLUTIndex(time_t t) const
|
||||
inline LUTIndex toLUTIndex(Time t) const
|
||||
{
|
||||
return findIndex(t);
|
||||
}
|
||||
@ -280,7 +285,7 @@ public:
|
||||
|
||||
/// Round down to start of monday.
|
||||
template <typename DateOrTime>
|
||||
inline time_t toFirstDayOfWeek(DateOrTime v) const
|
||||
inline Time toFirstDayOfWeek(DateOrTime v) const
|
||||
{
|
||||
const LUTIndex i = toLUTIndex(v);
|
||||
return lut[i - (lut[i].day_of_week - 1)].date;
|
||||
@ -295,7 +300,7 @@ public:
|
||||
|
||||
/// Round down to start of month.
|
||||
template <typename DateOrTime>
|
||||
inline time_t toFirstDayOfMonth(DateOrTime v) const
|
||||
inline Time toFirstDayOfMonth(DateOrTime v) const
|
||||
{
|
||||
const LUTIndex i = toLUTIndex(v);
|
||||
return lut[i - (lut[i].day_of_month - 1)].date;
|
||||
@ -332,13 +337,13 @@ public:
|
||||
}
|
||||
|
||||
template <typename DateOrTime>
|
||||
inline time_t toFirstDayOfQuarter(DateOrTime v) const
|
||||
inline Time toFirstDayOfQuarter(DateOrTime v) const
|
||||
{
|
||||
return toDate(toFirstDayOfQuarterIndex(v));
|
||||
}
|
||||
|
||||
/// Round down to start of year.
|
||||
inline time_t toFirstDayOfYear(time_t t) const
|
||||
inline Time toFirstDayOfYear(Time t) const
|
||||
{
|
||||
return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date;
|
||||
}
|
||||
@ -355,14 +360,14 @@ public:
|
||||
return toDayNum(toFirstDayNumOfYearIndex(v));
|
||||
}
|
||||
|
||||
inline time_t toFirstDayOfNextMonth(time_t t) const
|
||||
inline Time toFirstDayOfNextMonth(Time t) const
|
||||
{
|
||||
LUTIndex index = findIndex(t);
|
||||
index += 32 - lut[index].day_of_month;
|
||||
return lut[index - (lut[index].day_of_month - 1)].date;
|
||||
}
|
||||
|
||||
inline time_t toFirstDayOfPrevMonth(time_t t) const
|
||||
inline Time toFirstDayOfPrevMonth(Time t) const
|
||||
{
|
||||
LUTIndex index = findIndex(t);
|
||||
index -= lut[index].day_of_month;
|
||||
@ -389,16 +394,16 @@ public:
|
||||
|
||||
/** Round to start of day, then shift for specified amount of days.
|
||||
*/
|
||||
inline time_t toDateAndShift(time_t t, Int32 days) const
|
||||
inline Time toDateAndShift(Time t, Int32 days) const
|
||||
{
|
||||
return lut[findIndex(t) + days].date;
|
||||
}
|
||||
|
||||
inline time_t toTime(time_t t) const
|
||||
inline Time toTime(Time t) const
|
||||
{
|
||||
const LUTIndex index = findIndex(t);
|
||||
|
||||
time_t res = t - lut[index].date;
|
||||
Time res = t - lut[index].date;
|
||||
|
||||
if (res >= lut[index].time_at_offset_change())
|
||||
res += lut[index].amount_of_offset_change();
|
||||
@ -406,11 +411,11 @@ public:
|
||||
return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time.
|
||||
}
|
||||
|
||||
inline unsigned toHour(time_t t) const
|
||||
inline unsigned toHour(Time t) const
|
||||
{
|
||||
const LUTIndex index = findIndex(t);
|
||||
|
||||
time_t time = t - lut[index].date;
|
||||
Time time = t - lut[index].date;
|
||||
|
||||
if (time >= lut[index].time_at_offset_change())
|
||||
time += lut[index].amount_of_offset_change();
|
||||
@ -426,7 +431,7 @@ public:
|
||||
* then subtract the former from the latter to get the offset result.
|
||||
* The boundaries when meets DST(daylight saving time) change should be handled very carefully.
|
||||
*/
|
||||
inline time_t timezoneOffset(time_t t) const
|
||||
inline Time timezoneOffset(Time t) const
|
||||
{
|
||||
const LUTIndex index = findIndex(t);
|
||||
|
||||
@ -434,7 +439,7 @@ public:
|
||||
/// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin.
|
||||
/// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date,
|
||||
/// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now.
|
||||
time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400;
|
||||
Time res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400;
|
||||
|
||||
/// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder
|
||||
/// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward.
|
||||
@ -448,7 +453,7 @@ public:
|
||||
}
|
||||
|
||||
|
||||
inline unsigned toSecond(time_t t) const
|
||||
inline unsigned toSecond(Time t) const
|
||||
{
|
||||
auto res = t % 60;
|
||||
if (likely(res >= 0))
|
||||
@ -456,7 +461,7 @@ public:
|
||||
return res + 60;
|
||||
}
|
||||
|
||||
inline unsigned toMinute(time_t t) const
|
||||
inline unsigned toMinute(Time t) const
|
||||
{
|
||||
if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
|
||||
return (t / 60) % 60;
|
||||
@ -474,27 +479,27 @@ public:
|
||||
}
|
||||
|
||||
/// NOTE: Assuming timezone offset is a multiple of 15 minutes.
|
||||
inline time_t toStartOfMinute(time_t t) const { return roundDown(t, 60); }
|
||||
inline time_t toStartOfFiveMinute(time_t t) const { return roundDown(t, 300); }
|
||||
inline time_t toStartOfFifteenMinutes(time_t t) const { return roundDown(t, 900); }
|
||||
inline Time toStartOfMinute(Time t) const { return roundDown(t, 60); }
|
||||
inline Time toStartOfFiveMinute(Time t) const { return roundDown(t, 300); }
|
||||
inline Time toStartOfFifteenMinutes(Time t) const { return roundDown(t, 900); }
|
||||
|
||||
inline time_t toStartOfTenMinutes(time_t t) const
|
||||
inline Time toStartOfTenMinutes(Time t) const
|
||||
{
|
||||
if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
|
||||
return t / 600 * 600;
|
||||
|
||||
/// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate.
|
||||
Int64 date = find(t).date;
|
||||
Time date = find(t).date;
|
||||
return date + (t - date) / 600 * 600;
|
||||
}
|
||||
|
||||
/// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception.
|
||||
inline time_t toStartOfHour(time_t t) const
|
||||
inline Time toStartOfHour(Time t) const
|
||||
{
|
||||
if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
|
||||
return t / 3600 * 3600;
|
||||
|
||||
Int64 date = find(t).date;
|
||||
Time date = find(t).date;
|
||||
return date + (t - date) / 3600 * 3600;
|
||||
}
|
||||
|
||||
@ -506,11 +511,11 @@ public:
|
||||
* because the same calendar day starts/ends at different timestamps in different time zones)
|
||||
*/
|
||||
|
||||
inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; }
|
||||
inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; }
|
||||
inline Time fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; }
|
||||
inline Time fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; }
|
||||
|
||||
template <typename DateOrTime>
|
||||
inline time_t toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; }
|
||||
inline Time toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; }
|
||||
|
||||
template <typename DateOrTime>
|
||||
inline unsigned toMonth(DateOrTime v) const { return lut[toLUTIndex(v)].month; }
|
||||
@ -578,7 +583,7 @@ public:
|
||||
return toDayNum(toFirstDayNumOfISOYearIndex(v));
|
||||
}
|
||||
|
||||
inline time_t toFirstDayOfISOYear(time_t t) const
|
||||
inline Time toFirstDayOfISOYear(Time t) const
|
||||
{
|
||||
return lut[toFirstDayNumOfISOYearIndex(t)].date;
|
||||
}
|
||||
@ -773,7 +778,7 @@ public:
|
||||
}
|
||||
|
||||
/// We count all hour-length intervals, unrelated to offset changes.
|
||||
inline time_t toRelativeHourNum(time_t t) const
|
||||
inline Time toRelativeHourNum(Time t) const
|
||||
{
|
||||
if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
|
||||
return t / 3600;
|
||||
@ -784,18 +789,18 @@ public:
|
||||
}
|
||||
|
||||
template <typename DateOrTime>
|
||||
inline time_t toRelativeHourNum(DateOrTime v) const
|
||||
inline Time toRelativeHourNum(DateOrTime v) const
|
||||
{
|
||||
return toRelativeHourNum(lut[toLUTIndex(v)].date);
|
||||
}
|
||||
|
||||
inline time_t toRelativeMinuteNum(time_t t) const
|
||||
inline Time toRelativeMinuteNum(Time t) const
|
||||
{
|
||||
return (t + DATE_LUT_ADD) / 60 - (DATE_LUT_ADD / 60);
|
||||
}
|
||||
|
||||
template <typename DateOrTime>
|
||||
inline time_t toRelativeMinuteNum(DateOrTime v) const
|
||||
inline Time toRelativeMinuteNum(DateOrTime v) const
|
||||
{
|
||||
return toRelativeMinuteNum(lut[toLUTIndex(v)].date);
|
||||
}
|
||||
@ -842,14 +847,14 @@ public:
|
||||
return ExtendedDayNum(4 + (d - 4) / days * days);
|
||||
}
|
||||
|
||||
inline time_t toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const
|
||||
inline Time toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const
|
||||
{
|
||||
if (days == 1)
|
||||
return toDate(d);
|
||||
return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date;
|
||||
}
|
||||
|
||||
inline time_t toStartOfHourInterval(time_t t, UInt64 hours) const
|
||||
inline Time toStartOfHourInterval(Time t, UInt64 hours) const
|
||||
{
|
||||
if (hours == 1)
|
||||
return toStartOfHour(t);
|
||||
@ -867,7 +872,7 @@ public:
|
||||
const LUTIndex index = findIndex(t);
|
||||
const Values & values = lut[index];
|
||||
|
||||
time_t time = t - values.date;
|
||||
Time time = t - values.date;
|
||||
if (time >= values.time_at_offset_change())
|
||||
{
|
||||
/// Align to new hour numbers before rounding.
|
||||
@ -892,7 +897,7 @@ public:
|
||||
return values.date + time;
|
||||
}
|
||||
|
||||
inline time_t toStartOfMinuteInterval(time_t t, UInt64 minutes) const
|
||||
inline Time toStartOfMinuteInterval(Time t, UInt64 minutes) const
|
||||
{
|
||||
if (minutes == 1)
|
||||
return toStartOfMinute(t);
|
||||
@ -909,7 +914,7 @@ public:
|
||||
return roundDown(t, seconds);
|
||||
}
|
||||
|
||||
inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const
|
||||
inline Time toStartOfSecondInterval(Time t, UInt64 seconds) const
|
||||
{
|
||||
if (seconds == 1)
|
||||
return t;
|
||||
@ -934,14 +939,14 @@ public:
|
||||
return toDayNum(makeLUTIndex(year, month, day_of_month));
|
||||
}
|
||||
|
||||
inline time_t makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const
|
||||
inline Time makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const
|
||||
{
|
||||
return lut[makeLUTIndex(year, month, day_of_month)].date;
|
||||
}
|
||||
|
||||
/** Does not accept daylight saving time as argument: in case of ambiguity, it choose greater timestamp.
|
||||
*/
|
||||
inline time_t makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const
|
||||
inline Time makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const
|
||||
{
|
||||
size_t index = makeLUTIndex(year, month, day_of_month);
|
||||
UInt32 time_offset = hour * 3600 + minute * 60 + second;
|
||||
@ -969,7 +974,7 @@ public:
|
||||
return values.year * 10000 + values.month * 100 + values.day_of_month;
|
||||
}
|
||||
|
||||
inline time_t YYYYMMDDToDate(UInt32 num) const
|
||||
inline Time YYYYMMDDToDate(UInt32 num) const
|
||||
{
|
||||
return makeDate(num / 10000, num / 100 % 100, num % 100);
|
||||
}
|
||||
@ -1000,13 +1005,13 @@ public:
|
||||
TimeComponents time;
|
||||
};
|
||||
|
||||
inline DateComponents toDateComponents(time_t t) const
|
||||
inline DateComponents toDateComponents(Time t) const
|
||||
{
|
||||
const Values & values = getValues(t);
|
||||
return { values.year, values.month, values.day_of_month };
|
||||
}
|
||||
|
||||
inline DateTimeComponents toDateTimeComponents(time_t t) const
|
||||
inline DateTimeComponents toDateTimeComponents(Time t) const
|
||||
{
|
||||
const LUTIndex index = findIndex(t);
|
||||
const Values & values = lut[index];
|
||||
@ -1017,7 +1022,7 @@ public:
|
||||
res.date.month = values.month;
|
||||
res.date.day = values.day_of_month;
|
||||
|
||||
time_t time = t - values.date;
|
||||
Time time = t - values.date;
|
||||
if (time >= values.time_at_offset_change())
|
||||
time += values.amount_of_offset_change();
|
||||
|
||||
@ -1042,7 +1047,7 @@ public:
|
||||
}
|
||||
|
||||
|
||||
inline UInt64 toNumYYYYMMDDhhmmss(time_t t) const
|
||||
inline UInt64 toNumYYYYMMDDhhmmss(Time t) const
|
||||
{
|
||||
DateTimeComponents components = toDateTimeComponents(t);
|
||||
|
||||
@ -1055,7 +1060,7 @@ public:
|
||||
+ UInt64(components.date.year) * 10000000000;
|
||||
}
|
||||
|
||||
inline time_t YYYYMMDDhhmmssToTime(UInt64 num) const
|
||||
inline Time YYYYMMDDhhmmssToTime(UInt64 num) const
|
||||
{
|
||||
return makeDateTime(
|
||||
num / 10000000000,
|
||||
@ -1069,12 +1074,12 @@ public:
|
||||
/// Adding calendar intervals.
|
||||
/// Implementation specific behaviour when delta is too big.
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const
|
||||
inline NO_SANITIZE_UNDEFINED Time addDays(Time t, Int64 delta) const
|
||||
{
|
||||
const LUTIndex index = findIndex(t);
|
||||
const Values & values = lut[index];
|
||||
|
||||
time_t time = t - values.date;
|
||||
Time time = t - values.date;
|
||||
if (time >= values.time_at_offset_change())
|
||||
time += values.amount_of_offset_change();
|
||||
|
||||
@ -1086,7 +1091,7 @@ public:
|
||||
return lut[new_index].date + time;
|
||||
}
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED time_t addWeeks(time_t t, Int64 delta) const
|
||||
inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int64 delta) const
|
||||
{
|
||||
return addDays(t, delta * 7);
|
||||
}
|
||||
@ -1131,14 +1136,14 @@ public:
|
||||
|
||||
/// If resulting month has less deys than source month, then saturation can happen.
|
||||
/// Example: 31 Aug + 1 month = 30 Sep.
|
||||
inline time_t NO_SANITIZE_UNDEFINED addMonths(time_t t, Int64 delta) const
|
||||
inline Time NO_SANITIZE_UNDEFINED addMonths(Time t, Int64 delta) const
|
||||
{
|
||||
const auto result_day = addMonthsIndex(t, delta);
|
||||
|
||||
const LUTIndex index = findIndex(t);
|
||||
const Values & values = lut[index];
|
||||
|
||||
time_t time = t - values.date;
|
||||
Time time = t - values.date;
|
||||
if (time >= values.time_at_offset_change())
|
||||
time += values.amount_of_offset_change();
|
||||
|
||||
@ -1153,7 +1158,7 @@ public:
|
||||
return toDayNum(addMonthsIndex(d, delta));
|
||||
}
|
||||
|
||||
inline time_t NO_SANITIZE_UNDEFINED addQuarters(time_t t, Int64 delta) const
|
||||
inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int64 delta) const
|
||||
{
|
||||
return addMonths(t, delta * 3);
|
||||
}
|
||||
@ -1180,14 +1185,14 @@ public:
|
||||
}
|
||||
|
||||
/// Saturation can occur if 29 Feb is mapped to non-leap year.
|
||||
inline time_t addYears(time_t t, Int64 delta) const
|
||||
inline Time addYears(Time t, Int64 delta) const
|
||||
{
|
||||
auto result_day = addYearsIndex(t, delta);
|
||||
|
||||
const LUTIndex index = findIndex(t);
|
||||
const Values & values = lut[index];
|
||||
|
||||
time_t time = t - values.date;
|
||||
Time time = t - values.date;
|
||||
if (time >= values.time_at_offset_change())
|
||||
time += values.amount_of_offset_change();
|
||||
|
||||
@ -1203,7 +1208,7 @@ public:
|
||||
}
|
||||
|
||||
|
||||
inline std::string timeToString(time_t t) const
|
||||
inline std::string timeToString(Time t) const
|
||||
{
|
||||
DateTimeComponents components = toDateTimeComponents(t);
|
||||
|
||||
@ -1228,7 +1233,7 @@ public:
|
||||
return s;
|
||||
}
|
||||
|
||||
inline std::string dateToString(time_t t) const
|
||||
inline std::string dateToString(Time t) const
|
||||
{
|
||||
const Values & values = getValues(t);
|
||||
|
||||
|
||||
@ -1,9 +1,12 @@
|
||||
# This strings autochanged from release_lib.sh:
|
||||
SET(VERSION_REVISION 54452)
|
||||
# This variables autochanged by release_lib.sh:
|
||||
|
||||
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54453)
|
||||
SET(VERSION_MAJOR 21)
|
||||
SET(VERSION_MINOR 7)
|
||||
SET(VERSION_MINOR 8)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH 976ccc2e908ac3bc28f763bfea8134ea0a121b40)
|
||||
SET(VERSION_DESCRIBE v21.7.1.1-prestable)
|
||||
SET(VERSION_STRING 21.7.1.1)
|
||||
SET(VERSION_GITHASH fb895056568e26200629c7d19626e92d2dedc70d)
|
||||
SET(VERSION_DESCRIBE v21.8.1.1-prestable)
|
||||
SET(VERSION_STRING 21.8.1.1)
|
||||
# end of autochange
|
||||
|
||||
@ -33,44 +33,25 @@ macro(clickhouse_embed_binaries)
|
||||
message(FATAL_ERROR "The list of binary resources to embed may not be empty")
|
||||
endif()
|
||||
|
||||
# If cross-compiling, ensure we use the toolchain file and target the
|
||||
# actual target architecture
|
||||
if (CMAKE_CROSSCOMPILING)
|
||||
set(CROSS_COMPILE_FLAGS "--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${TOOLCHAIN_FILE}")
|
||||
else()
|
||||
set(CROSS_COMPILE_FLAGS "")
|
||||
endif()
|
||||
add_library("${EMBED_TARGET}" STATIC)
|
||||
set_target_properties("${EMBED_TARGET}" PROPERTIES LINKER_LANGUAGE C)
|
||||
|
||||
set(EMBED_TEMPLATE_FILE "${PROJECT_SOURCE_DIR}/programs/embed_binary.S.in")
|
||||
set(RESOURCE_OBJS)
|
||||
foreach(RESOURCE_FILE ${EMBED_RESOURCES})
|
||||
set(RESOURCE_OBJ "${RESOURCE_FILE}.o")
|
||||
list(APPEND RESOURCE_OBJS "${RESOURCE_OBJ}")
|
||||
|
||||
# Normalize the name of the resource
|
||||
foreach(RESOURCE_FILE ${EMBED_RESOURCES})
|
||||
set(ASSEMBLY_FILE_NAME "${RESOURCE_FILE}.S")
|
||||
set(BINARY_FILE_NAME "${RESOURCE_FILE}")
|
||||
|
||||
# Normalize the name of the resource.
|
||||
string(REGEX REPLACE "[\./-]" "_" SYMBOL_NAME "${RESOURCE_FILE}") # - must be last in regex
|
||||
string(REPLACE "+" "_PLUS_" SYMBOL_NAME "${SYMBOL_NAME}")
|
||||
set(ASSEMBLY_FILE_NAME "${RESOURCE_FILE}.S")
|
||||
|
||||
# Put the configured assembly file in the output directory.
|
||||
# This is so we can clean it up as usual, and we CD to the
|
||||
# source directory before compiling, so that the assembly
|
||||
# `.incbin` directive can find the file.
|
||||
# Generate the configured assembly file in the output directory.
|
||||
configure_file("${EMBED_TEMPLATE_FILE}" "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}" @ONLY)
|
||||
|
||||
# Generate the output object file by compiling the assembly, in the directory of
|
||||
# the sources so that the resource file may also be found
|
||||
add_custom_command(
|
||||
OUTPUT ${RESOURCE_OBJ}
|
||||
COMMAND cd "${EMBED_RESOURCE_DIR}" &&
|
||||
${CMAKE_C_COMPILER} "${CROSS_COMPILE_FLAGS}" -c -o
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}"
|
||||
)
|
||||
set_source_files_properties("${RESOURCE_OBJ}" PROPERTIES EXTERNAL_OBJECT true GENERATED true)
|
||||
endforeach()
|
||||
# Set the include directory for relative paths specified for `.incbin` directive.
|
||||
set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}" APPEND PROPERTY INCLUDE_DIRECTORIES "${EMBED_RESOURCE_DIR}")
|
||||
|
||||
add_library("${EMBED_TARGET}" STATIC ${RESOURCE_OBJS})
|
||||
set_target_properties("${EMBED_TARGET}" PROPERTIES LINKER_LANGUAGE C)
|
||||
target_sources("${EMBED_TARGET}" PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}")
|
||||
endforeach()
|
||||
endmacro()
|
||||
|
||||
@ -4,7 +4,6 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu")
|
||||
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc")
|
||||
get_filename_component (TOOLCHAIN_FILE "${CMAKE_TOOLCHAIN_FILE}" REALPATH)
|
||||
|
||||
# We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9.
|
||||
set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar" CACHE FILEPATH "" FORCE)
|
||||
|
||||
@ -26,7 +26,7 @@ if (NOT USE_INTERNAL_CCTZ_LIBRARY)
|
||||
set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ})
|
||||
set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ})
|
||||
endif()
|
||||
|
||||
|
||||
set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
|
||||
file(REMOVE ${SYSTEM_STORAGE_TZ_FILE})
|
||||
file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n")
|
||||
|
||||
2
contrib/libunwind
vendored
2
contrib/libunwind
vendored
@ -1 +1 @@
|
||||
Subproject commit a491c27b33109a842d577c0f7ac5f5f218859181
|
||||
Subproject commit cdcc3d8c6f6e80a0886082704a0902d61d8d3ffe
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
||||
clickhouse (21.7.1.1) unstable; urgency=low
|
||||
clickhouse (21.8.1.1) unstable; urgency=low
|
||||
|
||||
* Modified source code
|
||||
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Thu, 20 May 2021 22:23:29 +0300
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Mon, 28 Jun 2021 00:50:15 +0300
|
||||
|
||||
@ -12,7 +12,6 @@ mkdir root
|
||||
pushd root
|
||||
mkdir lib lib64 etc tmp root
|
||||
cp ${BUILD_DIR}/programs/clickhouse .
|
||||
cp ${SRC_DIR}/programs/server/{config,users}.xml .
|
||||
cp /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} lib
|
||||
cp /lib64/ld-linux-x86-64.so.2 lib64
|
||||
cp /etc/resolv.conf ./etc
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.7.1.*
|
||||
ARG version=21.8.1.*
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -72,7 +72,7 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
|
||||
&& cd .. \
|
||||
&& rm -rf apple-libtapi
|
||||
|
||||
# Build and install tools for cross-linking to Darwin
|
||||
# Build and install tools for cross-linking to Darwin (x86-64)
|
||||
RUN git clone https://github.com/tpoechtrager/cctools-port.git \
|
||||
&& cd cctools-port/cctools \
|
||||
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
|
||||
@ -81,8 +81,17 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
|
||||
&& cd ../.. \
|
||||
&& rm -rf cctools-port
|
||||
|
||||
# Download toolchain for Darwin
|
||||
RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz
|
||||
# Build and install tools for cross-linking to Darwin (aarch64)
|
||||
RUN git clone https://github.com/tpoechtrager/cctools-port.git \
|
||||
&& cd cctools-port/cctools \
|
||||
&& ./configure --prefix=/cctools --with-libtapi=/cctools \
|
||||
--target=aarch64-apple-darwin \
|
||||
&& make install \
|
||||
&& cd ../.. \
|
||||
&& rm -rf cctools-port
|
||||
|
||||
# Download toolchain and SDK for Darwin
|
||||
RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
|
||||
|
||||
# Download toolchain for ARM
|
||||
# It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling.
|
||||
|
||||
@ -3,7 +3,9 @@
|
||||
set -x -e
|
||||
|
||||
mkdir -p build/cmake/toolchain/darwin-x86_64
|
||||
tar xJf MacOSX10.15.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
|
||||
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
|
||||
|
||||
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
|
||||
|
||||
mkdir -p build/cmake/toolchain/linux-aarch64
|
||||
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
|
||||
|
||||
@ -58,6 +58,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache
|
||||
def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
|
||||
CLANG_PREFIX = "clang"
|
||||
DARWIN_SUFFIX = "-darwin"
|
||||
DARWIN_ARM_SUFFIX = "-darwin-aarch64"
|
||||
ARM_SUFFIX = "-aarch64"
|
||||
FREEBSD_SUFFIX = "-freebsd"
|
||||
|
||||
@ -66,9 +67,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
|
||||
|
||||
is_clang = compiler.startswith(CLANG_PREFIX)
|
||||
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
|
||||
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
|
||||
is_cross_arm = compiler.endswith(ARM_SUFFIX)
|
||||
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
|
||||
is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
|
||||
is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd
|
||||
|
||||
# Explicitly use LLD with Clang by default.
|
||||
# Don't force linker for cross-compilation.
|
||||
@ -82,6 +84,13 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
|
||||
cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
|
||||
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
|
||||
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
|
||||
elif is_cross_darwin_arm:
|
||||
cc = compiler[:-len(DARWIN_ARM_SUFFIX)]
|
||||
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
|
||||
cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/aarch64-apple-darwin-install_name_tool")
|
||||
cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib")
|
||||
cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
|
||||
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake")
|
||||
elif is_cross_arm:
|
||||
cc = compiler[:-len(ARM_SUFFIX)]
|
||||
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
|
||||
@ -185,8 +194,8 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
||||
parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd",
|
||||
"gcc-10"), default="clang-11")
|
||||
parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
|
||||
"clang-11-freebsd", "gcc-10"), default="clang-11")
|
||||
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
|
||||
parser.add_argument("--unbundled", action="store_true")
|
||||
parser.add_argument("--split-binary", action="store_true")
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.7.1.*
|
||||
ARG version=21.8.1.*
|
||||
ARG gosu_ver=1.10
|
||||
|
||||
# set non-empty deb_location_url url to create a docker image
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.7.1.*
|
||||
ARG version=21.8.1.*
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y apt-transport-https dirmngr && \
|
||||
|
||||
@ -113,6 +113,7 @@ function start_server
|
||||
echo "ClickHouse server pid '$server_pid' started and responded"
|
||||
|
||||
echo "
|
||||
set follow-fork-mode child
|
||||
handle all noprint
|
||||
handle SIGSEGV stop print
|
||||
handle SIGBUS stop print
|
||||
|
||||
@ -103,6 +103,7 @@ function fuzz
|
||||
kill -0 $server_pid
|
||||
|
||||
echo "
|
||||
set follow-fork-mode child
|
||||
handle all noprint
|
||||
handle SIGSEGV stop print
|
||||
handle SIGBUS stop print
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
# docker build -t yandex/clickhouse-integration-test .
|
||||
FROM yandex/clickhouse-test-base
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get -y install \
|
||||
tzdata \
|
||||
@ -20,7 +22,9 @@ RUN apt-get update \
|
||||
krb5-user \
|
||||
iproute2 \
|
||||
lsof \
|
||||
g++
|
||||
g++ \
|
||||
default-jre
|
||||
|
||||
RUN rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
@ -30,6 +34,19 @@ RUN apt-get clean
|
||||
# Install MySQL ODBC driver
|
||||
RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so
|
||||
|
||||
# Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper.
|
||||
# ZooKeeper is not started by default, but consumes some space in containers.
|
||||
# 777 perms used to allow anybody to start/stop ZooKeeper
|
||||
ENV ZOOKEEPER_VERSION='3.6.3'
|
||||
RUN curl -O "https://mirrors.estointernet.in/apache/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz"
|
||||
RUN tar -zxvf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && mv apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/zookeeper && chmod -R 777 /opt/zookeeper && rm apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz
|
||||
RUN echo $'tickTime=2500 \n\
|
||||
tickTime=2500 \n\
|
||||
dataDir=/zookeeper \n\
|
||||
clientPort=2181 \n\
|
||||
maxClientCnxns=80' > /opt/zookeeper/conf/zoo.cfg
|
||||
RUN mkdir /zookeeper && chmod -R 777 /zookeeper
|
||||
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
|
||||
@ -409,10 +409,10 @@ create view right_query_log as select *
|
||||
'$(cat "right-query-log.tsv.columns")');
|
||||
|
||||
create view query_logs as
|
||||
select 0 version, query_id, ProfileEvents.Names, ProfileEvents.Values,
|
||||
select 0 version, query_id, ProfileEvents.keys, ProfileEvents.values,
|
||||
query_duration_ms, memory_usage from left_query_log
|
||||
union all
|
||||
select 1 version, query_id, ProfileEvents.Names, ProfileEvents.Values,
|
||||
select 1 version, query_id, ProfileEvents.keys, ProfileEvents.values,
|
||||
query_duration_ms, memory_usage from right_query_log
|
||||
;
|
||||
|
||||
@ -424,7 +424,7 @@ create table query_run_metric_arrays engine File(TSV, 'analyze/query-run-metric-
|
||||
with (
|
||||
-- sumMapState with the list of all keys with '-0.' values. Negative zero is because
|
||||
-- sumMap removes keys with positive zeros.
|
||||
with (select groupUniqArrayArray(ProfileEvents.Names) from query_logs) as all_names
|
||||
with (select groupUniqArrayArray(ProfileEvents.keys) from query_logs) as all_names
|
||||
select arrayReduce('sumMapState', [(all_names, arrayMap(x->-0., all_names))])
|
||||
) as all_metrics
|
||||
select test, query_index, version, query_id,
|
||||
@ -433,8 +433,8 @@ create table query_run_metric_arrays engine File(TSV, 'analyze/query-run-metric-
|
||||
[
|
||||
all_metrics,
|
||||
arrayReduce('sumMapState',
|
||||
[(ProfileEvents.Names,
|
||||
arrayMap(x->toFloat64(x), ProfileEvents.Values))]
|
||||
[(ProfileEvents.keys,
|
||||
arrayMap(x->toFloat64(x), ProfileEvents.values))]
|
||||
),
|
||||
arrayReduce('sumMapState', [(
|
||||
['client_time', 'server_time', 'memory_usage'],
|
||||
@ -1005,7 +1005,7 @@ create table unstable_run_metrics engine File(TSVWithNamesAndTypes,
|
||||
'unstable-run-metrics.$version.rep') as
|
||||
select
|
||||
test, query_index, query_id,
|
||||
ProfileEvents.Values value, ProfileEvents.Names metric
|
||||
ProfileEvents.values value, ProfileEvents.keys metric
|
||||
from query_log array join ProfileEvents
|
||||
join unstable_query_runs using (query_id)
|
||||
;
|
||||
@ -1280,7 +1280,7 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
|
||||
then
|
||||
echo Database for test results is not specified, will not upload them.
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
set +x # Don't show password in the log
|
||||
client=(clickhouse-client
|
||||
|
||||
@ -561,7 +561,7 @@ if args.report == 'main':
|
||||
# Don't show mildly unstable queries, only the very unstable ones we
|
||||
# treat as errors.
|
||||
if very_unstable_queries:
|
||||
if very_unstable_queries > 3:
|
||||
if very_unstable_queries > 5:
|
||||
error_tests += very_unstable_queries
|
||||
status = 'failure'
|
||||
message_array.append(str(very_unstable_queries) + ' unstable')
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2094
|
||||
# shellcheck disable=SC2086
|
||||
|
||||
set -x
|
||||
|
||||
@ -37,6 +39,17 @@ function stop()
|
||||
|
||||
function start()
|
||||
{
|
||||
# Rename existing log file - it will be more convenient to read separate files for separate server runs.
|
||||
if [ -f '/var/log/clickhouse-server/clickhouse-server.log' ]
|
||||
then
|
||||
log_file_counter=1
|
||||
while [ -f "/var/log/clickhouse-server/clickhouse-server.log.${log_file_counter}" ]
|
||||
do
|
||||
log_file_counter=$((log_file_counter + 1))
|
||||
done
|
||||
mv '/var/log/clickhouse-server/clickhouse-server.log' "/var/log/clickhouse-server/clickhouse-server.log.${log_file_counter}"
|
||||
fi
|
||||
|
||||
counter=0
|
||||
until clickhouse-client --query "SELECT 1"
|
||||
do
|
||||
@ -55,6 +68,7 @@ function start()
|
||||
done
|
||||
|
||||
echo "
|
||||
set follow-fork-mode child
|
||||
handle all noprint
|
||||
handle SIGSEGV stop print
|
||||
handle SIGBUS stop print
|
||||
@ -140,7 +154,11 @@ zgrep -Fa "########################################" /test_output/* > /dev/null
|
||||
&& echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
# Put logs into /test_output/
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz
|
||||
for log_file in /var/log/clickhouse-server/clickhouse-server.log*
|
||||
do
|
||||
pigz < "${log_file}" > /test_output/"$(basename ${log_file})".gz
|
||||
done
|
||||
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
mv /var/log/clickhouse-server/stderr.log /test_output/
|
||||
tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
|
||||
|
||||
6
docs/_includes/install/arm.sh
Normal file
6
docs/_includes/install/arm.sh
Normal file
@ -0,0 +1,6 @@
|
||||
# ARM (AArch64) build works on Amazon Graviton, Oracle Cloud, Huawei Cloud ARM machines.
|
||||
# The support for AArch64 is pre-production ready.
|
||||
|
||||
wget 'https://builds.clickhouse.tech/master/aarch64/clickhouse'
|
||||
chmod a+x ./clickhouse
|
||||
sudo ./clickhouse install
|
||||
3
docs/_includes/install/freebsd.sh
Normal file
3
docs/_includes/install/freebsd.sh
Normal file
@ -0,0 +1,3 @@
|
||||
wget 'https://builds.clickhouse.tech/master/freebsd/clickhouse'
|
||||
chmod a+x ./clickhouse
|
||||
sudo ./clickhouse install
|
||||
3
docs/_includes/install/mac-arm.sh
Normal file
3
docs/_includes/install/mac-arm.sh
Normal file
@ -0,0 +1,3 @@
|
||||
wget 'https://builds.clickhouse.tech/master/macos-aarch64/clickhouse'
|
||||
chmod a+x ./clickhouse
|
||||
./clickhouse
|
||||
3
docs/_includes/install/mac-x86.sh
Normal file
3
docs/_includes/install/mac-x86.sh
Normal file
@ -0,0 +1,3 @@
|
||||
wget 'https://builds.clickhouse.tech/master/macos/clickhouse'
|
||||
chmod a+x ./clickhouse
|
||||
./clickhouse
|
||||
@ -33,7 +33,7 @@ Reboot.
|
||||
|
||||
``` bash
|
||||
brew update
|
||||
brew install cmake ninja libtool gettext llvm gcc
|
||||
brew install cmake ninja libtool gettext llvm gcc binutils
|
||||
```
|
||||
|
||||
## Checkout ClickHouse Sources {#checkout-clickhouse-sources}
|
||||
|
||||
@ -49,6 +49,7 @@ When working with the `MaterializeMySQL` database engine, [ReplacingMergeTree](.
|
||||
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
|
||||
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
||||
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
|
||||
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
|
||||
| STRING | [String](../../sql-reference/data-types/string.md) |
|
||||
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
|
||||
| BLOB | [String](../../sql-reference/data-types/string.md) |
|
||||
|
||||
@ -0,0 +1,53 @@
|
||||
---
|
||||
toc_priority: 12
|
||||
toc_title: ExternalDistributed
|
||||
---
|
||||
|
||||
# ExternalDistributed {#externaldistributed}
|
||||
|
||||
The `ExternalDistributed` engine allows to perform `SELECT` queries on data that is stored on a remote servers MySQL or PostgreSQL. Accepts [MySQL](../../../engines/table-engines/integrations/mysql.md) or [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) engines as an argument so sharding is possible.
|
||||
|
||||
## Creating a Table {#creating-a-table}
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
(
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
|
||||
...
|
||||
) ENGINE = ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password');
|
||||
```
|
||||
|
||||
See a detailed description of the [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) query.
|
||||
|
||||
The table structure can differ from the original table structure:
|
||||
|
||||
- Column names should be the same as in the original table, but you can use just some of these columns and in any order.
|
||||
- Column types may differ from those in the original table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
- `engine` — The table engine `MySQL` or `PostgreSQL`.
|
||||
- `host:port` — MySQL or PostgreSQL server address.
|
||||
- `database` — Remote database name.
|
||||
- `table` — Remote table name.
|
||||
- `user` — User name.
|
||||
- `password` — User password.
|
||||
|
||||
## Implementation Details {#implementation-details}
|
||||
|
||||
Supports multiple replicas that must be listed by `|` and shards must be listed by `,`. For example:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) ENGINE = ExternalDistributed('MySQL', `mysql{1|2}:3306,mysql{3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');
|
||||
```
|
||||
|
||||
When specifying replicas, one of the available replicas is selected for each of the shards when reading. If the connection fails, the next replica is selected, and so on for all the replicas. If the connection attempt fails for all the replicas, the attempt is repeated the same way several times.
|
||||
|
||||
You can specify any number of shards and any number of replicas for each shard.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [MySQL table engine](../../../engines/table-engines/integrations/mysql.md)
|
||||
- [PostgreSQL table engine](../../../engines/table-engines/integrations/postgresql.md)
|
||||
- [Distributed table engine](../../../engines/table-engines/special/distributed.md)
|
||||
@ -28,8 +28,8 @@ See a detailed description of the [CREATE TABLE](../../../sql-reference/statemen
|
||||
The table structure can differ from the original MySQL table structure:
|
||||
|
||||
- Column names should be the same as in the original MySQL table, but you can use just some of these columns and in any order.
|
||||
- Column types may differ from those in the original MySQL table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is true, if false - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types.
|
||||
- Column types may differ from those in the original MySQL table. ClickHouse tries to [cast](../../../engines/database-engines/mysql.md#data_types-support) values to the ClickHouse data types.
|
||||
- The [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) setting defines how to handle Nullable columns. Default value: 1. If 0, the table function does not make Nullable columns and inserts default values instead of nulls. This is also applicable for NULL values inside arrays.
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
@ -55,6 +55,12 @@ Simple `WHERE` clauses such as `=, !=, >, >=, <, <=` are executed on the MySQL s
|
||||
|
||||
The rest of the conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to MySQL finishes.
|
||||
|
||||
Supports multiple replicas that must be listed by `|`. For example:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_replicas (id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL(`mysql{2|3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');
|
||||
```
|
||||
|
||||
## Usage Example {#usage-example}
|
||||
|
||||
Table in MySQL:
|
||||
|
||||
@ -29,7 +29,7 @@ The table structure can differ from the source table structure:
|
||||
|
||||
- Column names should be the same as in the source table, but you can use just some of these columns and in any order.
|
||||
- Column types may differ from those in the source table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is true, if false - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types.
|
||||
- The [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) setting defines how to handle Nullable columns. Default value: 1. If 0, the table function does not make Nullable columns and inserts default values instead of nulls. This is also applicable for NULL values inside arrays.
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
|
||||
@ -23,8 +23,8 @@ See a detailed description of the [CREATE TABLE](../../../sql-reference/statemen
|
||||
The table structure can differ from the original PostgreSQL table structure:
|
||||
|
||||
- Column names should be the same as in the original PostgreSQL table, but you can use just some of these columns and in any order.
|
||||
- Column types may differ from those in the original PostgreSQL table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is 1, if 0 - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types.
|
||||
- Column types may differ from those in the original PostgreSQL table. ClickHouse tries to [cast](../../../engines/database-engines/postgresql.md#data_types-support) values to the ClickHouse data types.
|
||||
- The [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) setting defines how to handle Nullable columns. Default value: 1. If 0, the table function does not make Nullable columns and inserts default values instead of nulls. This is also applicable for NULL values inside arrays.
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
@ -49,6 +49,12 @@ PostgreSQL `Array` types are converted into ClickHouse arrays.
|
||||
|
||||
!!! info "Note"
|
||||
Be careful - in PostgreSQL an array data, created like a `type_name[]`, may contain multi-dimensional arrays of different dimensions in different table rows in same column. But in ClickHouse it is only allowed to have multidimensional arrays of the same count of dimensions in all table rows in same column.
|
||||
|
||||
Supports multiple replicas that must be listed by `|`. For example:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_replicas (id UInt32, name String) ENGINE = PostgreSQL(`postgres{2|3|4}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword');
|
||||
```
|
||||
|
||||
Replicas priority for PostgreSQL dictionary source is supported. The bigger the number in map, the less the priority. The highest priority is `0`.
|
||||
|
||||
|
||||
@ -65,7 +65,7 @@ By checking the row count:
|
||||
|
||||
Query:
|
||||
|
||||
``` sq;
|
||||
``` sql
|
||||
SELECT count() FROM recipes;
|
||||
```
|
||||
|
||||
|
||||
@ -94,11 +94,11 @@ For production environments, it’s recommended to use the latest `stable`-versi
|
||||
|
||||
To run ClickHouse inside Docker follow the guide on [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Those images use official `deb` packages inside.
|
||||
|
||||
### Single Binary
|
||||
### Single Binary {#from-single-binary}
|
||||
|
||||
You can install ClickHouse on Linux using single portable binary from the latest commit of the `master` branch: [https://builds.clickhouse.tech/master/amd64/clickhouse].
|
||||
You can install ClickHouse on Linux using a single portable binary from the latest commit of the `master` branch: [https://builds.clickhouse.tech/master/amd64/clickhouse].
|
||||
|
||||
```
|
||||
``` bash
|
||||
curl -O 'https://builds.clickhouse.tech/master/amd64/clickhouse' && chmod a+x clickhouse
|
||||
sudo ./clickhouse install
|
||||
```
|
||||
@ -107,9 +107,10 @@ sudo ./clickhouse install
|
||||
|
||||
For non-Linux operating systems and for AArch64 CPU arhitecture, ClickHouse builds are provided as a cross-compiled binary from the latest commit of the `master` branch (with a few hours delay).
|
||||
|
||||
- [macOS](https://builds.clickhouse.tech/master/macos/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [FreeBSD](https://builds.clickhouse.tech/master/freebsd/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/freebsd/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [AArch64](https://builds.clickhouse.tech/master/aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/aarch64/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [MacOS x86_64](https://builds.clickhouse.tech/master/macos/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [MacOS Aarch64 (Apple Silicon)](https://builds.clickhouse.tech/master/macos-aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos-aarch64/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [FreeBSD x86_64](https://builds.clickhouse.tech/master/freebsd/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/freebsd/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [Linux AArch64](https://builds.clickhouse.tech/master/aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/aarch64/clickhouse' && chmod a+x ./clickhouse`
|
||||
|
||||
After downloading, you can use the `clickhouse client` to connect to the server, or `clickhouse local` to process local data.
|
||||
|
||||
|
||||
@ -498,7 +498,7 @@ Return a message.
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<http_handlers>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
``` bash
|
||||
|
||||
@ -110,7 +110,7 @@ toc_title: Adopters
|
||||
| <a href="https://www.semrush.com/" class="favicon">SEMrush</a> | Marketing | Main product | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) |
|
||||
| <a href="https://sentry.io/" class="favicon">Sentry</a> | Software Development | Main product | — | — | [Blog Post in English, May 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) |
|
||||
| <a href="https://seo.do/" class="favicon">seo.do</a> | Analytics | Main product | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) |
|
||||
| <a href="http://www.sgk.gov.tr/wps/portal/sgk/tr" class="favicon">SGK</a> | Goverment Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) |
|
||||
| <a href="http://www.sgk.gov.tr/wps/portal/sgk/tr" class="favicon">SGK</a> | Government Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) |
|
||||
| <a href="http://english.sina.com/index.html" class="favicon">Sina</a> | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) |
|
||||
| <a href="https://smi2.ru/" class="favicon">SMI2</a> | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) |
|
||||
| <a href="https://www.spark.co.nz/" class="favicon">Spark New Zealand</a> | Telecommunications | Security Operations | — | — | [Blog Post, Feb 2020](https://blog.n0p.me/2020/02/2020-02-05-dnsmonster/) |
|
||||
@ -154,5 +154,6 @@ toc_title: Adopters
|
||||
| <a href="https://www.hydrolix.io/" class="favicon">Hydrolix</a> | Cloud data platform | Main product | — | — | [Documentation](https://docs.hydrolix.io/guide/query) |
|
||||
| <a href="https://www.argedor.com/en/clickhouse/" class="favicon">Argedor</a> | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) |
|
||||
| <a href="https://signoz.io/" class="favicon">SigNoz</a> | Observability Platform | Main Product | — | — | [Source code](https://github.com/SigNoz/signoz) |
|
||||
| <a href="https://chelpipegroup.com/" class="favicon">ChelPipe Group</a> | Analytics | — | — | — | [Blog post, June 2021](https://vc.ru/trade/253172-tyazhelomu-proizvodstvu-user-friendly-sayt-internet-magazin-trub-dlya-chtpz) |
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/introduction/adopters/) <!--hide-->
|
||||
|
||||
@ -379,7 +379,7 @@ Default value: `1`.
|
||||
|
||||
## insert_null_as_default {#insert_null_as_default}
|
||||
|
||||
Enables or disables the insertion of [default values](../../sql-reference/statements/create/table.md#create-default-values) instead of [NULL](../../sql-reference/syntax.md#null-literal) into columns with not [nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) data type.
|
||||
Enables or disables the insertion of [default values](../../sql-reference/statements/create/table.md#create-default-values) instead of [NULL](../../sql-reference/syntax.md#null-literal) into columns with not [nullable](../../sql-reference/data-types/nullable.md#data_type-nullable) data type.
|
||||
If column type is not nullable and this setting is disabled, then inserting `NULL` causes an exception. If column type is nullable, then `NULL` values are inserted as is, regardless of this setting.
|
||||
|
||||
This setting is applicable to [INSERT ... SELECT](../../sql-reference/statements/insert-into.md#insert_query_insert-select) queries. Note that `SELECT` subqueries may be concatenated with `UNION ALL` clause.
|
||||
@ -1182,7 +1182,7 @@ Possible values:
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
**Additional Info**
|
||||
**Additional Info**
|
||||
|
||||
This setting is useful for replicated tables with a sampling key. A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases:
|
||||
|
||||
@ -1194,21 +1194,22 @@ This setting is useful for replicated tables with a sampling key. A query may be
|
||||
!!! warning "Warning"
|
||||
This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md#max_parallel_replica-subqueries) for more details.
|
||||
|
||||
## compile {#compile}
|
||||
## compile_expressions {#compile-expressions}
|
||||
|
||||
Enable compilation of queries. By default, 0 (disabled).
|
||||
Enables or disables compilation of frequently used simple functions and operators to native code with LLVM at runtime.
|
||||
|
||||
The compilation is only used for part of the query-processing pipeline: for the first stage of aggregation (GROUP BY).
|
||||
If this portion of the pipeline was compiled, the query may run faster due to the deployment of short cycles and inlining aggregate function calls. The maximum performance improvement (up to four times faster in rare cases) is seen for queries with multiple simple aggregate functions. Typically, the performance gain is insignificant. In very rare cases, it may slow down query execution.
|
||||
Possible values:
|
||||
|
||||
## min_count_to_compile {#min-count-to-compile}
|
||||
- 0 — Disabled.
|
||||
- 1 — Enabled.
|
||||
|
||||
How many times to potentially use a compiled chunk of code before running compilation. By default, 3.
|
||||
For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values starting with 1. Compilation normally takes about 5-10 seconds.
|
||||
If the value is 1 or more, compilation occurs asynchronously in a separate thread. The result will be used as soon as it is ready, including queries that are currently running.
|
||||
Default value: `1`.
|
||||
|
||||
Compiled code is required for each different combination of aggregate functions used in the query and the type of keys in the GROUP BY clause.
|
||||
The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they do not use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted.
|
||||
## min_count_to_compile_expression {#min-count-to-compile-expression}
|
||||
|
||||
Minimum count of executing same expression before it is get compiled.
|
||||
|
||||
Default value: `3`.
|
||||
|
||||
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
|
||||
|
||||
@ -1558,7 +1559,7 @@ Possible values:
|
||||
|
||||
- 0 — Disabled (final query processing is done on the initiator node).
|
||||
- 1 - Do not merge aggregation states from different servers for distributed query processing (query completelly processed on the shard, initiator only proxy the data), can be used in case it is for certain that there are different keys on different shards.
|
||||
- 2 - Same as `1` but applies `ORDER BY` and `LIMIT` (it is not possilbe when the query processed completelly on the remote node, like for `distributed_group_by_no_merge=1`) on the initiator (can be used for queries with `ORDER BY` and/or `LIMIT`).
|
||||
- 2 - Same as `1` but applies `ORDER BY` and `LIMIT` (it is not possible when the query processed completelly on the remote node, like for `distributed_group_by_no_merge=1`) on the initiator (can be used for queries with `ORDER BY` and/or `LIMIT`).
|
||||
|
||||
**Example**
|
||||
|
||||
@ -1622,7 +1623,7 @@ Possible values:
|
||||
|
||||
Default value: 0
|
||||
|
||||
## optimize_skip_unused_shards_rewrite_in {#optimize-skip-unused-shardslrewrite-in}
|
||||
## optimize_skip_unused_shards_rewrite_in {#optimize-skip-unused-shards-rewrite-in}
|
||||
|
||||
Rewrite IN in query for remote shards to exclude values that does not belong to the shard (requires optimize_skip_unused_shards).
|
||||
|
||||
@ -1727,6 +1728,28 @@ Possible values:
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## optimize_functions_to_subcolumns {#optimize-functions-to-subcolumns}
|
||||
|
||||
Enables or disables optimization by transforming some functions to reading subcolumns. This reduces the amount of data to read.
|
||||
|
||||
These functions can be transformed:
|
||||
|
||||
- [length](../../sql-reference/functions/array-functions.md#array_functions-length) to read the [size0](../../sql-reference/data-types/array.md#array-size) subcolumn.
|
||||
- [empty](../../sql-reference/functions/array-functions.md#function-empty) to read the [size0](../../sql-reference/data-types/array.md#array-size) subcolumn.
|
||||
- [notEmpty](../../sql-reference/functions/array-functions.md#function-notempty) to read the [size0](../../sql-reference/data-types/array.md#array-size) subcolumn.
|
||||
- [isNull](../../sql-reference/operators/index.md#operator-is-null) to read the [null](../../sql-reference/data-types/nullable.md#finding-null) subcolumn.
|
||||
- [isNotNull](../../sql-reference/operators/index.md#is-not-null) to read the [null](../../sql-reference/data-types/nullable.md#finding-null) subcolumn.
|
||||
- [count](../../sql-reference/aggregate-functions/reference/count.md) to read the [null](../../sql-reference/data-types/nullable.md#finding-null) subcolumn.
|
||||
- [mapKeys](../../sql-reference/functions/tuple-map-functions.md#mapkeys) to read the [keys](../../sql-reference/data-types/map.md#map-subcolumns) subcolumn.
|
||||
- [mapValues](../../sql-reference/functions/tuple-map-functions.md#mapvalues) to read the [values](../../sql-reference/data-types/map.md#map-subcolumns) subcolumn.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Optimization disabled.
|
||||
- 1 — Optimization enabled.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
## distributed_replica_error_half_life {#settings-distributed_replica_error_half_life}
|
||||
|
||||
- Type: seconds
|
||||
@ -1802,6 +1825,27 @@ Possible values:
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## distributed_directory_monitor_split_batch_on_failure {#distributed_directory_monitor_split_batch_on_failure}
|
||||
|
||||
Enables/disables splitting batches on failures.
|
||||
|
||||
Sometimes sending particular batch to the remote shard may fail, because of some complex pipeline after (i.e. `MATERIALIZED VIEW` with `GROUP BY`) due to `Memory limit exceeded` or similar errors. In this case, retrying will not help (and this will stuck distributed sends for the table) but sending files from that batch one by one may succeed INSERT.
|
||||
|
||||
So installing this setting to `1` will disable batching for such batches (i.e. temporary disables `distributed_directory_monitor_batch_inserts` for failed batches).
|
||||
|
||||
Possible values:
|
||||
|
||||
- 1 — Enabled.
|
||||
- 0 — Disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
!!! note "Note"
|
||||
This setting also affects broken batches (that may appears because of abnormal server (machine) termination and no `fsync_after_insert`/`fsync_directories` for [Distributed](../../engines/table-engines/special/distributed.md) table engine).
|
||||
|
||||
!!! warning "Warning"
|
||||
You should not rely on automatic batch splitting, since this may hurt performance.
|
||||
|
||||
## os_thread_priority {#setting-os-thread-priority}
|
||||
|
||||
Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core.
|
||||
@ -2085,7 +2129,7 @@ Default value: 128.
|
||||
|
||||
## background_fetches_pool_size {#background_fetches_pool_size}
|
||||
|
||||
Sets the number of threads performing background fetches for [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. For production usage with frequent small insertions or slow ZooKeeper cluster is recomended to use default value.
|
||||
Sets the number of threads performing background fetches for [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. For production usage with frequent small insertions or slow ZooKeeper cluster is recommended to use default value.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -2672,7 +2716,7 @@ Default value: `0`.
|
||||
## aggregate_functions_null_for_empty {#aggregate_functions_null_for_empty}
|
||||
|
||||
Enables or disables rewriting all aggregate functions in a query, adding [-OrNull](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-ornull) suffix to them. Enable it for SQL standard compatibility.
|
||||
It is implemented via query rewrite (similar to [count_distinct_implementation](#settings-count_distinct_implementation) setting) to get consistent results for distributed queries.
|
||||
It is implemented via query rewrite (similar to [count_distinct_implementation](#settings-count_distinct_implementation) setting) to get consistent results for distributed queries.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -2856,7 +2900,7 @@ Default value: `0`.
|
||||
|
||||
## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
|
||||
|
||||
Adds a modifier `SYNC` to all `DROP` and `DETACH` queries.
|
||||
Adds a modifier `SYNC` to all `DROP` and `DETACH` queries.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -2962,7 +3006,7 @@ Enables or disables using the original column names instead of aliases in query
|
||||
Possible values:
|
||||
|
||||
- 0 — The column name is substituted with the alias.
|
||||
- 1 — The column name is not substituted with the alias.
|
||||
- 1 — The column name is not substituted with the alias.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
@ -3075,7 +3119,7 @@ SELECT
|
||||
sum(a),
|
||||
sumCount(b).1,
|
||||
sumCount(b).2,
|
||||
(sumCount(b).1) / (sumCount(b).2)
|
||||
(sumCount(b).1) / (sumCount(b).2)
|
||||
FROM fuse_tbl
|
||||
```
|
||||
|
||||
@ -3144,4 +3188,17 @@ SETTINGS index_granularity = 8192 │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) <!-- hide -->
|
||||
## external_table_functions_use_nulls {#external-table-functions-use-nulls}
|
||||
|
||||
Defines how [mysql](../../sql-reference/table-functions/mysql.md), [postgresql](../../sql-reference/table-functions/postgresql.md) and [odbc](../../sql-reference/table-functions/odbc.md)] table functions use Nullable columns.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — The table function explicitly uses Nullable columns.
|
||||
- 1 — The table function implicitly uses Nullable columns.
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
**Usage**
|
||||
|
||||
If the setting is set to `0`, the table function does not make Nullable columns and inserts default values instead of NULL. This is also applicable for NULL values inside arrays.
|
||||
|
||||
39
docs/en/operations/system-tables/data_skipping_indices.md
Normal file
39
docs/en/operations/system-tables/data_skipping_indices.md
Normal file
@ -0,0 +1,39 @@
|
||||
# system.data_skipping_indices {#system-data-skipping-indices}
|
||||
|
||||
Contains information about existing data skipping indices in all the tables.
|
||||
|
||||
Columns:
|
||||
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — Database name.
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Index name.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Index type.
|
||||
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression used to calculate the index.
|
||||
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of granules in the block.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
database: default
|
||||
table: user_actions
|
||||
name: clicks_idx
|
||||
type: minmax
|
||||
expr: clicks
|
||||
granularity: 1
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
database: default
|
||||
table: users
|
||||
name: contacts_null_idx
|
||||
type: minmax
|
||||
expr: assumeNotNull(contacts_null)
|
||||
granularity: 1
|
||||
```
|
||||
@ -34,14 +34,14 @@ initial_port: 47588
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
elapsed: 0.000582537
|
||||
is_cancelled: 0
|
||||
read_rows: 0
|
||||
@ -53,12 +53,10 @@ memory_usage: 0
|
||||
peak_memory_usage: 0
|
||||
query: SELECT * from system.processes LIMIT 10 FORMAT Vertical;
|
||||
thread_ids: [67]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','ContextLock','RWLockAcquiredReadLocks']
|
||||
ProfileEvents.Values: [1,1,36,1,10,1,89,16,1]
|
||||
Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage']
|
||||
Settings.Values: ['0','in_order','1','10000000000']
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
|
||||
1 rows in set. Elapsed: 0.002 sec.
|
||||
1 rows in set. Elapsed: 0.002 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/processes) <!--hide-->
|
||||
|
||||
@ -84,12 +84,10 @@ Columns:
|
||||
- `forwarded_for` ([String](../../sql-reference/data-types/string.md)) — HTTP header `X-Forwarded-For` passed in the HTTP query.
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — The `quota key` specified in the [quotas](../../operations/quotas.md) setting (see `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
|
||||
- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/array.md)) — ProfileEvents that measure different metrics. The description of them could be found in the table [system.events](../../operations/system-tables/events.md#system_tables-events)
|
||||
- `Settings` ([Map(String, String)](../../sql-reference/data-types/array.md)) — Settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
|
||||
- `log_comment` ([String](../../sql-reference/data-types/string.md)) — Log comment. It can be set to arbitrary string no longer than [max_query_size](../../operations/settings/settings.md#settings-max_query_size). An empty string if it is not defined.
|
||||
- `thread_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Thread ids that are participating in query execution.
|
||||
- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics. The description of them could be found in the table [system.events](../../operations/system-tables/events.md#system_tables-events)
|
||||
- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Values of metrics that are listed in the `ProfileEvents.Names` column.
|
||||
- `Settings.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1.
|
||||
- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — Values of settings that are listed in the `Settings.Names` column.
|
||||
- `used_aggregate_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `aggregate functions`, which were used during query execution.
|
||||
- `used_aggregate_function_combinators` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `aggregate functions combinators`, which were used during query execution.
|
||||
- `used_database_engines` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `database engines`, which were used during query execution.
|
||||
@ -109,68 +107,49 @@ SELECT * FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDa
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
type: QueryFinish
|
||||
event_date: 2021-03-18
|
||||
event_time: 2021-03-18 20:54:18
|
||||
event_time_microseconds: 2021-03-18 20:54:18.676686
|
||||
query_start_time: 2021-03-18 20:54:18
|
||||
query_start_time_microseconds: 2021-03-18 20:54:18.673934
|
||||
query_duration_ms: 2
|
||||
read_rows: 100
|
||||
read_bytes: 800
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 2
|
||||
result_bytes: 4858
|
||||
memory_usage: 0
|
||||
current_database: default
|
||||
query: SELECT uniqArray([1, 1, 2]), SUBSTRING('Hello, world', 7, 5), flatten([[[BIT_AND(123)]], [[mod(3, 2)], [CAST('1' AS INTEGER)]]]), week(toDate('2000-12-05')), CAST(arrayJoin([NULL, NULL]) AS Nullable(TEXT)), avgOrDefaultIf(number, number % 2), sumOrNull(number), toTypeName(sumOrNull(number)), countIf(toDate('2000-12-05') + number as d, toDayOfYear(d) % 2) FROM numbers(100)
|
||||
normalized_query_hash: 17858008518552525706
|
||||
query_kind: Select
|
||||
databases: ['_table_function']
|
||||
tables: ['_table_function.numbers']
|
||||
columns: ['_table_function.numbers.number']
|
||||
exception_code: 0
|
||||
type: QueryStart
|
||||
event_date: 2020-09-11
|
||||
event_time: 2020-09-11 10:08:17
|
||||
event_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_start_time: 2020-09-11 10:08:17
|
||||
query_start_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_duration_ms: 0
|
||||
read_rows: 0
|
||||
read_bytes: 0
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 0
|
||||
result_bytes: 0
|
||||
memory_usage: 0
|
||||
current_database: default
|
||||
query: INSERT INTO test1 VALUES
|
||||
exception_code: 0
|
||||
exception:
|
||||
stack_trace:
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 58f3d392-0fa0-4663-ae1d-29917a1a9c9c
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 37486
|
||||
initial_user: default
|
||||
initial_query_id: 58f3d392-0fa0-4663-ae1d-29917a1a9c9c
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 37486
|
||||
interface: 1
|
||||
os_user: sevirov
|
||||
client_hostname: clickhouse.ru-central1.internal
|
||||
client_name: ClickHouse
|
||||
client_revision: 54447
|
||||
client_version_major: 21
|
||||
client_version_minor: 4
|
||||
client_version_patch: 1
|
||||
http_method: 0
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 33452
|
||||
initial_user: default
|
||||
initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 33452
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
http_referer:
|
||||
forwarded_for:
|
||||
quota_key:
|
||||
revision: 54449
|
||||
log_comment:
|
||||
thread_ids: [587,11939]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','TableFunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes']
|
||||
ProfileEvents.Values: [1,1,36,1,10,2,1048680,1,4096,36,1,110,100,800,77,1,3137,1476,1101,8,2577,8192]
|
||||
Settings.Names: ['load_balancing','max_memory_usage']
|
||||
Settings.Values: ['random','10000000000']
|
||||
used_aggregate_functions: ['groupBitAnd','avg','sum','count','uniq']
|
||||
used_aggregate_function_combinators: ['OrDefault','If','OrNull','Array']
|
||||
used_database_engines: []
|
||||
used_data_type_families: ['String','Array','Int32','Nullable']
|
||||
used_dictionaries: []
|
||||
used_formats: []
|
||||
used_functions: ['toWeek','CAST','arrayFlatten','toTypeName','toDayOfYear','addDays','array','toDate','modulo','substring','plus']
|
||||
used_storages: []
|
||||
used_table_functions: ['numbers']
|
||||
revision: 54440
|
||||
thread_ids: []
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -58,8 +58,7 @@ Columns:
|
||||
- `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request.
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — The “quota key” specified in the [quotas](../../operations/quotas.md) setting (see `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
|
||||
- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events).
|
||||
- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
|
||||
- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/array.md)) — ProfileEvents that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events).
|
||||
|
||||
**Example**
|
||||
|
||||
@ -98,17 +97,16 @@ initial_port: 33452
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
revision: 54440
|
||||
ProfileEvents.Names: ['Query','InsertQuery','FileOpen','WriteBufferFromFileDescriptorWrite','WriteBufferFromFileDescriptorWriteBytes','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','FunctionExecute','CreatedWriteBufferOrdinary','DiskWriteElapsedMicroseconds','NetworkReceiveElapsedMicroseconds','NetworkSendElapsedMicroseconds','InsertedRows','InsertedBytes','SelectedRows','SelectedBytes','MergeTreeDataWriterRows','MergeTreeDataWriterUncompressedBytes','MergeTreeDataWriterCompressedBytes','MergeTreeDataWriterBlocks','MergeTreeDataWriterBlocksAlreadySorted','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSReadChars','OSWriteChars']
|
||||
ProfileEvents.Values: [1,1,11,11,591,148,3,71,29,6533808,1,11,72,18,47,1,12,1,12,1,12,189,1,1,10,2,70853,2748,49,2747,45056,422,1520]
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
@ -31,6 +31,8 @@ ClickHouse supports the `COUNT(DISTINCT ...)` syntax. The behavior of this const
|
||||
|
||||
The `SELECT count() FROM table` query is not optimized, because the number of entries in the table is not stored separately. It chooses a small column from the table and counts the number of values in it.
|
||||
|
||||
However `SELECT count(nullable_column) FROM table` query can be optimized by enabling the [optimize_functions_to_subcolumns](../../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [null](../../../sql-reference/data-types/nullable.md#finding-null) subcolumn instead of reading and processing the whole column data. The query `SELECT count(n) FROM table` transforms to `SELECT sum(NOT n.null) FROM table`.
|
||||
|
||||
**Examples**
|
||||
|
||||
Example 1:
|
||||
|
||||
@ -4,14 +4,14 @@ The `median*` functions are the aliases for the corresponding `quantile*` functi
|
||||
|
||||
Functions:
|
||||
|
||||
- `median` — Alias for [quantile](#quantile).
|
||||
- `medianDeterministic` — Alias for [quantileDeterministic](#quantiledeterministic).
|
||||
- `medianExact` — Alias for [quantileExact](#quantileexact).
|
||||
- `medianExactWeighted` — Alias for [quantileExactWeighted](#quantileexactweighted).
|
||||
- `medianTiming` — Alias for [quantileTiming](#quantiletiming).
|
||||
- `medianTimingWeighted` — Alias for [quantileTimingWeighted](#quantiletimingweighted).
|
||||
- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest).
|
||||
- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted).
|
||||
- `median` — Alias for [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile).
|
||||
- `medianDeterministic` — Alias for [quantileDeterministic](../../../sql-reference/aggregate-functions/reference/quantiledeterministic.md#quantiledeterministic).
|
||||
- `medianExact` — Alias for [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexact).
|
||||
- `medianExactWeighted` — Alias for [quantileExactWeighted](../../../sql-reference/aggregate-functions/reference/quantileexactweighted.md#quantileexactweighted).
|
||||
- `medianTiming` — Alias for [quantileTiming](../../../sql-reference/aggregate-functions/reference/quantiletiming.md#quantiletiming).
|
||||
- `medianTimingWeighted` — Alias for [quantileTimingWeighted](../../../sql-reference/aggregate-functions/reference/quantiletimingweighted.md#quantiletimingweighted).
|
||||
- `medianTDigest` — Alias for [quantileTDigest](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md#quantiletdigest).
|
||||
- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](../../../sql-reference/aggregate-functions/reference/quantiletdigestweighted.md#quantiletdigestweighted).
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -2,7 +2,9 @@
|
||||
toc_priority: 202
|
||||
---
|
||||
|
||||
# quantileExact {#quantileexact}
|
||||
# quantileExact Functions {#quantileexact-functions}
|
||||
|
||||
## quantileExact {#quantileexact}
|
||||
|
||||
Exactly computes the [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
@ -49,7 +51,7 @@ Result:
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
# quantileExactLow {#quantileexactlow}
|
||||
## quantileExactLow {#quantileexactlow}
|
||||
|
||||
Similar to `quantileExact`, this computes the exact [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
@ -66,7 +68,7 @@ SELECT quantileExactLow(0.1)(number) FROM numbers(10)
|
||||
│ 1 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
|
||||
|
||||
**Syntax**
|
||||
@ -107,12 +109,11 @@ Result:
|
||||
│ 4 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
# quantileExactHigh {#quantileexacthigh}
|
||||
## quantileExactHigh {#quantileexacthigh}
|
||||
|
||||
Similar to `quantileExact`, this computes the exact [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
All the passed values are combined into an array, which is then fully sorted,
|
||||
to get the exact value. The sorting [algorithm's](https://en.cppreference.com/w/cpp/algorithm/sort) complexity is `O(N·log(N))`, where `N = std::distance(first, last)` comparisons.
|
||||
All the passed values are combined into an array, which is then fully sorted, to get the exact value. The sorting [algorithm's](https://en.cppreference.com/w/cpp/algorithm/sort) complexity is `O(N·log(N))`, where `N = std::distance(first, last)` comparisons.
|
||||
|
||||
The return value depends on the quantile level and the number of elements in the selection, i.e. if the level is 0.5, then the function returns the higher median value for an even number of elements and the middle median value for an odd number of elements. Median is calculated similarly to the [median_high](https://docs.python.org/3/library/statistics.html#statistics.median_high) implementation which is used in python. For all other levels, the element at the index corresponding to the value of `level * size_of_array` is returned.
|
||||
|
||||
@ -158,6 +159,111 @@ Result:
|
||||
│ 5 │
|
||||
└───────────────────────────┘
|
||||
```
|
||||
|
||||
## quantileExactExclusive {#quantileexactexclusive}
|
||||
|
||||
Exactly computes the [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` memory, where `n` is a number of values that were passed. However, for a small number of values, the function is very effective.
|
||||
|
||||
This function is equivalent to [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba) Excel function, ([type R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)).
|
||||
|
||||
When using multiple `quantileExactExclusive` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantilesexactexclusive) function.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
quantileExactExclusive(level)(expr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `level` — Level of quantile. Optional. Possible values: (0, 1) — bounds not included. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median). [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Quantile of the specified level.
|
||||
|
||||
Type:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
|
||||
- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantileExactExclusive(0.6)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─quantileExactExclusive(0.6)(x)─┐
|
||||
│ 599.6 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
## quantileExactInclusive {#quantileexactinclusive}
|
||||
|
||||
Exactly computes the [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` memory, where `n` is a number of values that were passed. However, for a small number of values, the function is very effective.
|
||||
|
||||
This function is equivalent to [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed) Excel function, ([type R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)).
|
||||
|
||||
When using multiple `quantileExactInclusive` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantilesexactinclusive) function.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
quantileExactInclusive(level)(expr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `level` — Level of quantile. Optional. Possible values: [0, 1] — bounds included. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median). [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Quantile of the specified level.
|
||||
|
||||
Type:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
|
||||
- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantileExactInclusive(0.6)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─quantileExactInclusive(0.6)(x)─┐
|
||||
│ 599.4 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [median](../../../sql-reference/aggregate-functions/reference/median.md#median)
|
||||
|
||||
@ -2,8 +2,114 @@
|
||||
toc_priority: 201
|
||||
---
|
||||
|
||||
# quantiles {#quantiles}
|
||||
# quantiles Functions {#quantiles-functions}
|
||||
|
||||
## quantiles {#quantiles}
|
||||
|
||||
Syntax: `quantiles(level1, level2, …)(x)`
|
||||
|
||||
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
||||
|
||||
## quantilesExactExclusive {#quantilesexactexclusive}
|
||||
|
||||
Exactly computes the [quantiles](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` memory, where `n` is a number of values that were passed. However, for a small number of values, the function is very effective.
|
||||
|
||||
This function is equivalent to [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba) Excel function, ([type R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)).
|
||||
|
||||
Works more efficiently with sets of levels than [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
quantilesExactExclusive(level1, level2, ...)(expr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `level` — Levels of quantiles. Possible values: (0, 1) — bounds not included. [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Array](../../../sql-reference/data-types/array.md) of quantiles of the specified levels.
|
||||
|
||||
Type of array values:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
|
||||
- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantilesExactExclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─quantilesExactExclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x)─┐
|
||||
│ [249.25,499.5,749.75,899.9,949.9499999999999,989.99,998.999] │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## quantilesExactInclusive {#quantilesexactinclusive}
|
||||
|
||||
Exactly computes the [quantiles](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence.
|
||||
|
||||
To get exact value, all the passed values are combined into an array, which is then partially sorted. Therefore, the function consumes `O(n)` memory, where `n` is a number of values that were passed. However, for a small number of values, the function is very effective.
|
||||
|
||||
This function is equivalent to [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed) Excel function, ([type R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)).
|
||||
|
||||
Works more efficiently with sets of levels than [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantilesexactinclusive).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
quantilesExactInclusive(level1, level2, ...)(expr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `level` — Levels of quantiles. Possible values: [0, 1] — bounds included. [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Array](../../../sql-reference/data-types/array.md) of quantiles of the specified levels.
|
||||
|
||||
Type of array values:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
|
||||
- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantilesExactInclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─quantilesExactInclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x)─┐
|
||||
│ [249.75,499.5,749.25,899.1,949.05,989.01,998.001] │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -8,6 +8,7 @@ toc_title: Map(key, value)
|
||||
`Map(key, value)` data type stores `key:value` pairs.
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md).
|
||||
- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
@ -75,6 +76,36 @@ SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
## Map.keys and Map.values Subcolumns {#map-subcolumns}
|
||||
|
||||
To optimize `Map` column processing, in some cases you can use the `keys` and `values` subcolumns instead of reading the whole column.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE t_map (`a` Map(String, UInt64)) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_map VALUES (map('key1', 1, 'key2', 2, 'key3', 3));
|
||||
|
||||
SELECT a.keys FROM t_map;
|
||||
|
||||
SELECT a.values FROM t_map;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─a.keys─────────────────┐
|
||||
│ ['key1','key2','key3'] │
|
||||
└────────────────────────┘
|
||||
|
||||
┌─a.values─┐
|
||||
│ [1,2,3] │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function
|
||||
|
||||
@ -57,7 +57,7 @@ In this case, ClickHouse can reload the dictionary earlier if the dictionary con
|
||||
When updating the dictionaries, the ClickHouse server applies different logic depending on the type of [source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md):
|
||||
|
||||
- For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated.
|
||||
- For MySQL source, the time of modification is checked using a `SHOW TABLE STATUS` query (in case of MySQL 8 you need to disable meta-information caching in MySQL by `set global information_schema_stats_expiry=0`.
|
||||
- For MySQL source, the time of modification is checked using a `SHOW TABLE STATUS` query (in case of MySQL 8 you need to disable meta-information caching in MySQL by `set global information_schema_stats_expiry=0`).
|
||||
- Dictionaries from other sources are updated every time by default.
|
||||
|
||||
For other sources (ODBC, PostgreSQL, ClickHouse, etc), you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps:
|
||||
@ -88,13 +88,13 @@ SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source wher
|
||||
|
||||
For `Cache`, `ComplexKeyCache`, `SSDCache`, and `SSDComplexKeyCache` dictionaries both synchronious and asynchronious updates are supported.
|
||||
|
||||
It is also possible for `Flat`, `Hashed`, `ComplexKeyHashed` dictionaries to only request data that was changed after previous update. If `update_field` is specified in as part of dictionary source configuration value of previous update time in seconds will be added to data request. Depends of source type Executable, HTTP, MySQL, PostgreSQL, ClickHouse, ODBC different logic will be applied to `update_field` before request data from external source.
|
||||
It is also possible for `Flat`, `Hashed`, `ComplexKeyHashed` dictionaries to only request data that was changed after the previous update. If `update_field` is specified as part of the dictionary source configuration, value of the previous update time in seconds will be added to the data request. Depends on source type (Executable, HTTP, MySQL, PostgreSQL, ClickHouse, or ODBC) different logic will be applied to `update_field` before request data from an external source.
|
||||
|
||||
- If source is HTTP then `update_field` will be added as query parameter with last update time as parameter value.
|
||||
- If source is Executable then `update_field` will be added as executable script argument with last update time as argument value.
|
||||
- If source is ClickHouse, MySQL, PostgreSQL, ODBC there will be additional part of WHERE, where `update_field` is compared as greater or equal with last update time.
|
||||
- If the source is HTTP then `update_field` will be added as a query parameter with the last update time as the parameter value.
|
||||
- If the source is Executable then `update_field` will be added as an executable script argument with the last update time as the argument value.
|
||||
- If the source is ClickHouse, MySQL, PostgreSQL, ODBC there will be an additional part of `WHERE`, where `update_field` is compared as greater or equal with the last update time.
|
||||
|
||||
If `update_field` option is set. Additional option `update_lag` can be set. Value of `update_lag` option is subtracted from previous update time before request updated data.
|
||||
If `update_field` option is set, additional option `update_lag` can be set. Value of `update_lag` option is subtracted from previous update time before request updated data.
|
||||
|
||||
Example of settings:
|
||||
|
||||
@ -116,4 +116,4 @@ or
|
||||
...
|
||||
SOURCE(CLICKHOUSE(... update_field 'added_time' update_lag 15))
|
||||
...
|
||||
```
|
||||
```
|
||||
|
||||
@ -11,18 +11,24 @@ Returns 1 for an empty array, or 0 for a non-empty array.
|
||||
The result type is UInt8.
|
||||
The function also works for strings.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [size0](../../sql-reference/data-types/array.md#array-size) subcolumn instead of reading and processing the whole array column. The query `SELECT empty(arr) FROM table` transforms to `SELECT arr.size0 = 0 FROM TABLE`.
|
||||
|
||||
## notEmpty {#function-notempty}
|
||||
|
||||
Returns 0 for an empty array, or 1 for a non-empty array.
|
||||
The result type is UInt8.
|
||||
The function also works for strings.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [size0](../../sql-reference/data-types/array.md#array-size) subcolumn instead of reading and processing the whole array column. The query `SELECT notEmpty(arr) FROM table` transforms to `SELECT arr.size0 != 0 FROM TABLE`.
|
||||
|
||||
## length {#array_functions-length}
|
||||
|
||||
Returns the number of items in the array.
|
||||
The result type is UInt64.
|
||||
The function also works for strings.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [size0](../../sql-reference/data-types/array.md#array-size) subcolumn instead of reading and processing the whole array column. The query `SELECT length(arr) FROM table` transforms to `SELECT arr.size0 FROM TABLE`.
|
||||
|
||||
## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64}
|
||||
|
||||
## emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64}
|
||||
|
||||
@ -220,6 +220,8 @@ Result:
|
||||
|
||||
Returns all keys from the `map` parameter.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [keys](../../sql-reference/data-types/map.md#map-subcolumns) subcolumn instead of reading and processing the whole column data. The query `SELECT mapKeys(m) FROM table` transforms to `SELECT m.keys FROM table`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
@ -261,6 +263,8 @@ Result:
|
||||
|
||||
Returns all values from the `map` parameter.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [values](../../sql-reference/data-types/map.md#map-subcolumns) subcolumn instead of reading and processing the whole column data. The query `SELECT mapValues(m) FROM table` transforms to `SELECT m.values FROM table`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
|
||||
@ -283,6 +283,8 @@ ClickHouse supports the `IS NULL` and `IS NOT NULL` operators.
|
||||
- `0` otherwise.
|
||||
- For other values, the `IS NULL` operator always returns `0`.
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [null](../../sql-reference/data-types/nullable.md#finding-null) subcolumn instead of reading and processing the whole column data. The query `SELECT n IS NULL FROM table` transforms to `SELECT n.null FROM TABLE`.
|
||||
|
||||
<!-- -->
|
||||
|
||||
``` sql
|
||||
@ -313,3 +315,5 @@ SELECT * FROM t_null WHERE y IS NOT NULL
|
||||
│ 2 │ 3 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
Can be optimized by enabling the [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [null](../../sql-reference/data-types/nullable.md#finding-null) subcolumn instead of reading and processing the whole column data. The query `SELECT n IS NOT NULL FROM table` transforms to `SELECT NOT n.null FROM TABLE`.
|
||||
|
||||
@ -19,6 +19,8 @@ The following operations with [partitions](../../../engines/table-engines/merget
|
||||
- [UNFREEZE PARTITION](#alter_unfreeze-partition) — Removes a backup of a partition.
|
||||
- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server.
|
||||
- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume.
|
||||
- [UPDATE IN PARTITION](#update-in-partition) — Update data inside the partition by condition.
|
||||
- [DELETE IN PARTITION](#delete-in-partition) — Delete data inside the partition by condition.
|
||||
|
||||
<!-- -->
|
||||
|
||||
|
||||
@ -119,7 +119,7 @@ For manage uncompressed data cache parameters use following server level setting
|
||||
## DROP COMPILED EXPRESSION CACHE {#query_language-system-drop-compiled-expression-cache}
|
||||
|
||||
Reset the compiled expression cache. Used in development of ClickHouse and performance tests.
|
||||
Complied expression cache used when query/user/profile enable option [compile](../../operations/settings/settings.md#compile)
|
||||
Compiled expression cache used when query/user/profile enable option [compile-expressions](../../operations/settings/settings.md#compile-expressions)
|
||||
|
||||
## FLUSH LOGS {#query_language-system-flush_logs}
|
||||
|
||||
|
||||
@ -39,6 +39,18 @@ Simple `WHERE` clauses such as `=, !=, >, >=, <, <=` are currently executed on t
|
||||
|
||||
The rest of the conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to MySQL finishes.
|
||||
|
||||
Supports multiple replicas that must be listed by `|`. For example:
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql{1|2|3}:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
**Returned Value**
|
||||
|
||||
A table object with the same columns as the original MySQL table.
|
||||
|
||||
@ -43,8 +43,20 @@ PostgreSQL Array types converts into ClickHouse arrays.
|
||||
|
||||
!!! info "Note"
|
||||
Be careful, in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse it is only allowed to have multidimensional arrays of the same dimension in all rows.
|
||||
|
||||
Supports multiple replicas that must be listed by `|`. For example:
|
||||
|
||||
Supports replicas priority for PostgreSQL dictionary source. The bigger the number in map, the less the priority. The highest priority is `0`.
|
||||
```sql
|
||||
SELECT name FROM postgresql(`postgres{1|2|3}:5432`, 'postgres_database', 'postgres_table', 'user', 'password');
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```sql
|
||||
SELECT name FROM postgresql(`postgres1:5431|postgres2:5432`, 'postgres_database', 'postgres_table', 'user', 'password');
|
||||
```
|
||||
|
||||
Supports replicas priority for PostgreSQL dictionary source. The bigger the number in map, the less the priority. The highest priority is `0`.
|
||||
|
||||
**Examples**
|
||||
|
||||
|
||||
48
docs/en/sql-reference/table-functions/s3Cluster.md
Normal file
48
docs/en/sql-reference/table-functions/s3Cluster.md
Normal file
@ -0,0 +1,48 @@
|
||||
---
|
||||
toc_priority: 55
|
||||
toc_title: s3Cluster
|
||||
---
|
||||
|
||||
# s3Cluster Table Function {#s3Cluster-table-function}
|
||||
|
||||
Allows processing files from [Amazon S3](https://aws.amazon.com/s3/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster, discloses asterics in S3 file path, and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
|
||||
- `source` — URL to a file or a bunch of files. Supports following wildcards in readonly mode: `*`, `?`, `{'abc','def'}` and `{N..M}` where `N`, `M` — numbers, `abc`, `def` — strings. For more information see [Wildcards In Path](../../engines/table-engines/integrations/s3.md#wildcards-in-path).
|
||||
- `access_key_id` and `secret_access_key` — Keys that specify credentials to use with given endpoint. Optional.
|
||||
- `format` — The [format](../../interfaces/formats.md#formats) of the file.
|
||||
- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
|
||||
|
||||
**Returned value**
|
||||
|
||||
A table with the specified structure for reading or writing data in the specified file.
|
||||
|
||||
**Examples**
|
||||
|
||||
Select the data from all files in the cluster `cluster_simple`:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM s3Cluster('cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon);
|
||||
```
|
||||
|
||||
Count the total amount of rows in all files in the cluster `cluster_simple`:
|
||||
|
||||
``` sql
|
||||
SELECT count(*) FROM s3Cluster('cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))');
|
||||
```
|
||||
|
||||
!!! warning "Warning"
|
||||
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [S3 engine](../../engines/table-engines/integrations/s3.md)
|
||||
- [s3 table function](../../sql-reference/table-functions/s3.md)
|
||||
@ -7,7 +7,7 @@ toc_title: Security Changelog
|
||||
|
||||
### CVE-2019-15024 {#cve-2019-15024}
|
||||
|
||||
Аn attacker that has write access to ZooKeeper and who ican run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem.
|
||||
Аn attacker that has write access to ZooKeeper and who can run a custom server available from the network where ClickHouse runs, can create a custom-built malicious server that will act as a ClickHouse replica and register it in ZooKeeper. When another replica will fetch data part from the malicious replica, it can force clickhouse-server to write to arbitrary path on filesystem.
|
||||
|
||||
Credits: Eldar Zaitov of Yandex Information Security Team
|
||||
|
||||
|
||||
@ -478,7 +478,7 @@ max_alter_threads 2
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<http_handlers>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
``` bash
|
||||
|
||||
@ -817,22 +817,6 @@ load_balancing = first_or_random
|
||||
のための一貫性を異なる部分に同じデータを分割)、このオプションにしているときだけサンプリングキーを設定します。
|
||||
レプリカラグは制御されません。
|
||||
|
||||
## コンパイル {#compile}
|
||||
|
||||
を編集ます。 既定では、0(無効)です。
|
||||
|
||||
コンパイルは、クエリ処理パイプラインの一部にのみ使用されます。
|
||||
この部分のパイプラインのためのクエリを実行するアによる展開の短サイクルinlining集計機能。 複数の単純な集計関数を使用するクエリでは、最大のパフォーマンスの向上が見られます。 通常、性能は軽微であります。 非常に珍しい例で遅くなクエリを実行します。
|
||||
|
||||
## min_count_to_compile {#min-count-to-compile}
|
||||
|
||||
り方を潜在的に利用コチャンクのコードの実行前に作成する。 デフォルトでは3.
|
||||
For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values starting with 1. Compilation normally takes about 5-10 seconds.
|
||||
値が1以上の場合、コンパイルは別のスレッドで非同期に実行されます。 結果は、現在実行中のクエリを含め、準備が整うとすぐに使用されます。
|
||||
|
||||
コンパイルされたコードは、クエリで使用される集計関数とGROUP BY句内のキーの種類のそれぞれの異なる組み合わせに必要です。
|
||||
The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted.
|
||||
|
||||
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
|
||||
|
||||
値がtrueの場合、json\*Int64およびUInt64形式（ほとんどのJavaScript実装との互換性のため）を使用するときに整数が引用符で表示されます。
|
||||
|
||||
@ -625,10 +625,8 @@ ClickHouseはこのテーブルを作成します。 [query_log](server-configur
|
||||
- `quota_key` (String) — The “quota key” で指定される。 [クォータ](quotas.md) 設定(参照 `keyed`).
|
||||
- `revision` (UInt32) — ClickHouse revision.
|
||||
- `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution.
|
||||
- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [システムイベント](#system_tables-events)
|
||||
- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` 列。
|
||||
- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` パラメータは1。
|
||||
- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` 列。
|
||||
- `ProfileEvents` (Map(String, UInt64)) — ProfileEvents that measure different metrics. The description of them could be found in the table [システムイベント](#system_tables-events)
|
||||
- `Settings` (Map(String, String)) — Settings 列。
|
||||
|
||||
それぞれのクエリでは、一つまたは二つの行が `query_log` クエリのステータスに応じて、テーブル:
|
||||
|
||||
@ -698,8 +696,7 @@ ClickHouseはこのテーブルを作成します。 [query_thread_log](server-c
|
||||
- `http_user_agent` (String) — The `UserAgent` HTTP要求で渡されるヘッダー。
|
||||
- `quota_key` (String) — The “quota key” で指定される。 [クォータ](quotas.md) 設定(参照 `keyed`).
|
||||
- `revision` (UInt32) — ClickHouse revision.
|
||||
- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [システムイベント](#system_tables-events)
|
||||
- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` 列。
|
||||
- `ProfileEvents` (Map(String, UInt64)) — ProfileEvents that measure different metrics for this thread. The description of them could be found in the table [システムイベント](#system_tables-events)
|
||||
|
||||
既定では、ログは7.5秒間隔でテーブルに追加されます。 この間隔は [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) サーバ設定(参照 `flush_interval_milliseconds` 変数）。 ログをメモリバッファからテーブルに強制的にフラッシュするには、 `SYSTEM FLUSH LOGS` クエリ。
|
||||
|
||||
|
||||
@ -0,0 +1,53 @@
|
||||
---
|
||||
toc_priority: 12
|
||||
toc_title: ExternalDistributed
|
||||
---
|
||||
|
||||
# ExternalDistributed {#externaldistributed}
|
||||
|
||||
Движок `ExternalDistributed` позволяет выполнять запросы `SELECT` для таблиц на удаленном сервере MySQL или PostgreSQL. Принимает в качестве аргумента табличные движки [MySQL](../../../engines/table-engines/integrations/mysql.md) или [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md), поэтому возможно шардирование.
|
||||
|
||||
## Создание таблицы {#creating-a-table}
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
(
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
|
||||
...
|
||||
) ENGINE = ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password');
|
||||
```
|
||||
|
||||
Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query).
|
||||
|
||||
Структура таблицы может отличаться от структуры исходной таблицы:
|
||||
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице, но можно использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице. ClickHouse пытается [привести](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) значения к типам данных ClickHouse.
|
||||
|
||||
**Параметры движка**
|
||||
|
||||
- `engine` — табличный движок `MySQL` или `PostgreSQL`.
|
||||
- `host:port` — адрес сервера MySQL или PostgreSQL.
|
||||
- `database` — имя базы данных на сервере.
|
||||
- `table` — имя таблицы.
|
||||
- `user` — имя пользователя.
|
||||
- `password` — пароль пользователя.
|
||||
|
||||
## Особенности реализации {#implementation-details}
|
||||
|
||||
Поддерживает несколько реплик, которые должны быть перечислены через `|`, а шарды — через `,`. Например:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) ENGINE = ExternalDistributed('MySQL', `mysql{1|2}:3306,mysql{3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');
|
||||
```
|
||||
|
||||
При указании реплик для каждого из шардов при чтении выбирается одна из доступных реплик. Если соединиться не удалось, то выбирается следующая реплика, и так для всех реплик. Если попытка соединения не удалась для всех реплик, то сервер ClickHouse снова пытается соединиться с одной из реплик, перебирая их по кругу, и так несколько раз.
|
||||
|
||||
Вы можете указать любое количество шардов и любое количество реплик для каждого шарда.
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
- [Табличный движок MySQL](../../../engines/table-engines/integrations/mysql.md)
|
||||
- [Табличный движок PostgreSQL](../../../engines/table-engines/integrations/postgresql.md)
|
||||
- [Табличный движок Distributed](../../../engines/table-engines/special/distributed.md)
|
||||
@ -20,11 +20,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query).
|
||||
|
||||
Структура таблицы может отличаться от исходной структуры таблицы MySQL:
|
||||
Структура таблицы может отличаться от структуры исходной таблицы MySQL:
|
||||
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) значения к типам данных ClickHouse.
|
||||
- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов.
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице MySQL, но можно использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [привести](../../../engines/database-engines/mysql.md#data_types-support) значения к типам данных ClickHouse.
|
||||
- Настройка [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) определяет как обрабатывать Nullable столбцы. Значение по умолчанию: 1. Если значение 0, то табличная функция не делает Nullable столбцы, а вместо NULL выставляет значения по умолчанию для скалярного типа. Это также применимо для значений NULL внутри массивов.
|
||||
|
||||
**Параметры движка**
|
||||
|
||||
@ -50,6 +50,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
Остальные условия и ограничение выборки `LIMIT` будут выполнены в ClickHouse только после выполнения запроса к MySQL.
|
||||
|
||||
Поддерживает несколько реплик, которые должны быть перечислены через `|`. Например:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_replicas (id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL(`mysql{2|3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');
|
||||
```
|
||||
|
||||
## Пример использования {#primer-ispolzovaniia}
|
||||
|
||||
Таблица в MySQL:
|
||||
|
||||
@ -29,7 +29,7 @@ ENGINE = ODBC(connection_settings, external_database, external_table)
|
||||
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов аналогичных столбцов в исходной таблице. ClickHouse пытается [приводить](../../../engines/table-engines/integrations/odbc.md#type_conversion_function-cast) значения к типам данных ClickHouse.
|
||||
- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов.
|
||||
- Настройка [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) определяет как обрабатывать Nullable столбцы. Значение по умолчанию: 1. Если значение 0, то табличная функция не делает Nullable столбцы, а вместо NULL выставляет значения по умолчанию для скалярного типа. Это также применимо для значений NULL внутри массивов.
|
||||
|
||||
**Параметры движка**
|
||||
|
||||
|
||||
@ -20,19 +20,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query).
|
||||
|
||||
Структура таблицы может отличаться от исходной структуры таблицы PostgreSQL:
|
||||
Структура таблицы может отличаться от структуры исходной таблицы PostgreSQL:
|
||||
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице PostgreSQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице PostgreSQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов.
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице PostgreSQL, но можно использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице PostgreSQL. ClickHouse пытается [привести](../../../engines/database-engines/postgresql.md#data_types-support) значения к типам данных ClickHouse.
|
||||
- Настройка [external_table_functions_use_nulls](../../../operations/settings/settings.md#external-table-functions-use-nulls) определяет как обрабатывать Nullable столбцы. Значение по умолчанию: 1. Если значение 0, то табличная функция не делает Nullable столбцы, а вместо NULL выставляет значения по умолчанию для скалярного типа. Это также применимо для значений NULL внутри массивов.
|
||||
|
||||
**Параметры движка**
|
||||
|
||||
- `host:port` — адрес сервера PostgreSQL.
|
||||
- `database` — Имя базы данных на сервере PostgreSQL.
|
||||
- `table` — Имя таблицы.
|
||||
- `user` — Имя пользователя PostgreSQL.
|
||||
- `password` — Пароль пользователя PostgreSQL.
|
||||
- `database` — имя базы данных на сервере PostgreSQL.
|
||||
- `table` — имя таблицы.
|
||||
- `user` — имя пользователя PostgreSQL.
|
||||
- `password` — пароль пользователя PostgreSQL.
|
||||
- `schema` — имя схемы, если не используется схема по умолчанию. Необязательный аргумент.
|
||||
|
||||
## Особенности реализации {#implementation-details}
|
||||
@ -49,6 +49,12 @@ PostgreSQL массивы конвертируются в массивы ClickHo
|
||||
|
||||
!!! info "Внимание"
|
||||
Будьте внимательны, в PostgreSQL массивы, созданные как `type_name[]`, являются многомерными и могут содержать в себе разное количество измерений в разных строках одной таблицы. Внутри ClickHouse допустимы только многомерные массивы с одинаковым кол-вом измерений во всех строках таблицы.
|
||||
|
||||
Поддерживает несколько реплик, которые должны быть перечислены через `|`. Например:
|
||||
|
||||
```sql
|
||||
CREATE TABLE test_replicas (id UInt32, name String) ENGINE = PostgreSQL(`postgres{2|3|4}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword');
|
||||
```
|
||||
|
||||
При использовании словаря PostgreSQL поддерживается приоритет реплик. Чем больше номер реплики, тем ниже ее приоритет. Наивысший приоритет у реплики с номером `0`.
|
||||
|
||||
|
||||
@ -87,9 +87,18 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh
|
||||
|
||||
Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты.
|
||||
|
||||
### Из единого бинарного файла {#from-single-binary}
|
||||
|
||||
Для установки ClickHouse под Linux можно использовать единый переносимый бинарный файл из последнего коммита ветки `master`: [https://builds.clickhouse.tech/master/amd64/clickhouse].
|
||||
|
||||
``` bash
|
||||
curl -O 'https://builds.clickhouse.tech/master/amd64/clickhouse' && chmod a+x clickhouse
|
||||
sudo ./clickhouse install
|
||||
```
|
||||
|
||||
### Из исполняемых файлов для нестандартных окружений {#from-binaries-non-linux}
|
||||
|
||||
Для других операционных систем и архитектуры AArch64, сборки ClickHouse предоставляются в виде кросс-компилированного бинарника с последнего коммита ветки master (с задержкой в несколько часов).
|
||||
Для других операционных систем и архитектуры AArch64 сборки ClickHouse предоставляются в виде кросс-компилированного бинарного файла из последнего коммита ветки `master` (с задержкой в несколько часов).
|
||||
|
||||
- [macOS](https://builds.clickhouse.tech/master/macos/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos/clickhouse' && chmod a+x ./clickhouse`
|
||||
- [AArch64](https://builds.clickhouse.tech/master/aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/aarch64/clickhouse' && chmod a+x ./clickhouse`
|
||||
@ -97,9 +106,9 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh
|
||||
|
||||
После скачивания можно воспользоваться `clickhouse client` для подключения к серверу или `clickhouse local` для обработки локальных данных.
|
||||
|
||||
Чтобы установить ClickHouse в рамках всей системы (с необходимыми конфигурационными файлами, настройками пользователей и т.д.), выполните `sudo ./clickhouse install`. Затем выполните команды `clickhouse start` (чтобы запустить сервер) и `clickhouse-client` (чтобы подключиться к нему).
|
||||
Чтобы установить ClickHouse в рамках всей системы (с необходимыми конфигурационными файлами, настройками пользователей и т.д.), выполните `sudo ./clickhouse install`. Затем выполните команды `clickhouse start` (чтобы запустить сервер) и `clickhouse-client` (чтобы подключиться к нему).
|
||||
|
||||
Данные сборки не рекомендуются для использования в продакшене, так как они недостаточно тщательно протестированны. Также, в них присутствуют не все возможности ClickHouse.
|
||||
Данные сборки не рекомендуются для использования в рабочей среде, так как они недостаточно тщательно протестированы. Также в них присутствуют не все возможности ClickHouse.
|
||||
|
||||
### Из исходного кода {#from-sources}
|
||||
|
||||
|
||||
@ -499,7 +499,7 @@ max_alter_threads 2
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<http_handlers>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
``` bash
|
||||
|
||||
@ -348,7 +348,7 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2;
|
||||
## input_format_null_as_default {#settings-input-format-null-as-default}
|
||||
|
||||
Включает или отключает инициализацию [значениями по умолчанию](../../sql-reference/statements/create/table.md#create-default-values) ячеек с [NULL](../../sql-reference/syntax.md#null-literal), если тип данных столбца не позволяет [хранить NULL](../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
Если столбец не позволяет хранить `NULL` и эта настройка отключена, то вставка `NULL` приведет к возникновению исключения. Если столбец позволяет хранить `NULL`, то значения `NULL` вставляются независимо от этой настройки.
|
||||
Если столбец не позволяет хранить `NULL` и эта настройка отключена, то вставка `NULL` приведет к возникновению исключения. Если столбец позволяет хранить `NULL`, то значения `NULL` вставляются независимо от этой настройки.
|
||||
|
||||
Эта настройка используется для запросов [INSERT ... VALUES](../../sql-reference/statements/insert-into.md) для текстовых входных форматов.
|
||||
|
||||
@ -361,7 +361,7 @@ INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2;
|
||||
|
||||
## insert_null_as_default {#insert_null_as_default}
|
||||
|
||||
Включает или отключает вставку [значений по умолчанию](../../sql-reference/statements/create/table.md#create-default-values) вместо [NULL](../../sql-reference/syntax.md#null-literal) в столбцы, которые не позволяют [хранить NULL](../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
Включает или отключает вставку [значений по умолчанию](../../sql-reference/statements/create/table.md#create-default-values) вместо [NULL](../../sql-reference/syntax.md#null-literal) в столбцы, которые не позволяют [хранить NULL](../../sql-reference/data-types/nullable.md#data_type-nullable).
|
||||
Если столбец не позволяет хранить `NULL` и эта настройка отключена, то вставка `NULL` приведет к возникновению исключения. Если столбец позволяет хранить `NULL`, то значения `NULL` вставляются независимо от этой настройки.
|
||||
|
||||
Эта настройка используется для запросов [INSERT ... SELECT](../../sql-reference/statements/insert-into.md#insert_query_insert-select). При этом подзапросы `SELECT` могут объединяться с помощью `UNION ALL`.
|
||||
@ -1181,22 +1181,22 @@ load_balancing = round_robin
|
||||
!!! warning "Предупреждение"
|
||||
Параллельное выполнение запроса может привести к неверному результату, если в запросе есть объединение или подзапросы и при этом таблицы не удовлетворяют определенным требованиям. Подробности смотрите в разделе [Распределенные подзапросы и max_parallel_replicas](../../sql-reference/operators/in.md#max_parallel_replica-subqueries).
|
||||
|
||||
## compile_expressions {#compile-expressions}
|
||||
|
||||
## compile {#compile}
|
||||
Включает или выключает компиляцию часто используемых функций и операторов. Компиляция производится в нативный код платформы с помощью LLVM во время выполнения.
|
||||
|
||||
Включить компиляцию запросов. По умолчанию - 0 (выключено).
|
||||
Возможные значения:
|
||||
|
||||
Компиляция предусмотрена только для части конвейера обработки запроса - для первой стадии агрегации (GROUP BY).
|
||||
В случае, если эта часть конвейера была скомпилирована, запрос может работать быстрее, за счёт разворачивания коротких циклов и инлайнинга вызовов агрегатных функций. Максимальный прирост производительности (до четырёх раз в редких случаях) достигается на запросах с несколькими простыми агрегатными функциями. Как правило, прирост производительности незначителен. В очень редких случаях возможно замедление выполнения запроса.
|
||||
- 0 — компиляция выключена.
|
||||
- 1 — компиляция включена.
|
||||
|
||||
## min_count_to_compile {#min-count-to-compile}
|
||||
Значение по умолчанию: `1`.
|
||||
|
||||
После скольких раз, когда скомпилированный кусок кода мог пригодиться, выполнить его компиляцию. По умолчанию - 3.
|
||||
Для тестирования можно установить значение 0: компиляция выполняется синхронно, и запрос ожидает окончания процесса компиляции перед продолжением выполнения. Во всех остальных случаях используйте значения, начинающиеся с 1. Как правило, компиляция занимает по времени около 5-10 секунд.
|
||||
В случае, если значение равно 1 или больше, компиляция выполняется асинхронно, в отдельном потоке. При готовности результата, он сразу же будет использован, в том числе, уже выполняющимися в данный момент запросами.
|
||||
## min_count_to_compile_expression {#min-count-to-compile-expression}
|
||||
|
||||
Скомпилированный код требуется для каждого разного сочетания используемых в запросе агрегатных функций и вида ключей в GROUP BY.
|
||||
Результаты компиляции сохраняются в директории build в виде .so файлов. Количество результатов компиляции не ограничено, так как они не занимают много места. При перезапуске сервера, старые результаты будут использованы, за исключением случая обновления сервера - тогда старые результаты удаляются.
|
||||
Минимальное количество выполнений одного и того же выражения до его компиляции.
|
||||
|
||||
Значение по умолчанию: `3`.
|
||||
|
||||
## input_format_skip_unknown_fields {#input-format-skip-unknown-fields}
|
||||
|
||||
@ -1606,6 +1606,28 @@ ClickHouse генерирует исключение
|
||||
|
||||
Значение по умолчанию: 0.
|
||||
|
||||
## optimize_functions_to_subcolumns {#optimize-functions-to-subcolumns}
|
||||
|
||||
Включает или отключает оптимизацию путем преобразования некоторых функций к чтению подстолбцов, таким образом уменьшая объем данных для чтения.
|
||||
|
||||
Могут быть преобразованы следующие функции:
|
||||
|
||||
- [length](../../sql-reference/functions/array-functions.md#array_functions-length) к чтению подстолбца [size0](../../sql-reference/data-types/array.md#array-size) subcolumn.
|
||||
- [empty](../../sql-reference/functions/array-functions.md#function-empty) к чтению подстолбца [size0](../../sql-reference/data-types/array.md#array-size) subcolumn.
|
||||
- [notEmpty](../../sql-reference/functions/array-functions.md#function-notempty) к чтению подстолбца [size0](../../sql-reference/data-types/array.md#array-size).
|
||||
- [isNull](../../sql-reference/operators/index.md#operator-is-null) к чтению подстолбца [null](../../sql-reference/data-types/nullable.md#finding-null).
|
||||
- [isNotNull](../../sql-reference/operators/index.md#is-not-null) к чтению подстолбца [null](../../sql-reference/data-types/nullable.md#finding-null).
|
||||
- [count](../../sql-reference/aggregate-functions/reference/count.md) к чтению подстолбца [null](../../sql-reference/data-types/nullable.md#finding-null).
|
||||
- [mapKeys](../../sql-reference/functions/tuple-map-functions.md#mapkeys) к чтению подстолбца [keys](../../sql-reference/data-types/map.md#map-subcolumns).
|
||||
- [mapValues](../../sql-reference/functions/tuple-map-functions.md#mapvalues) к чтению подстолбца [values](../../sql-reference/data-types/map.md#map-subcolumns).
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- 0 — оптимизация отключена.
|
||||
- 1 — оптимизация включена.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
## distributed_replica_error_half_life {#settings-distributed_replica_error_half_life}
|
||||
|
||||
- Тип: секунды
|
||||
@ -2721,7 +2743,7 @@ SELECT * FROM test2;
|
||||
- 0 — запрос `INSERT` добавляет данные в конец файла после существующих.
|
||||
- 1 — `INSERT` удаляет имеющиеся в файле данные и замещает их новыми.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
## allow_experimental_geo_types {#allow-experimental-geo-types}
|
||||
|
||||
@ -2735,7 +2757,7 @@ SELECT * FROM test2;
|
||||
|
||||
## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
|
||||
|
||||
Добавляет модификатор `SYNC` ко всем запросам `DROP` и `DETACH`.
|
||||
Добавляет модификатор `SYNC` ко всем запросам `DROP` и `DETACH`.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
@ -2813,7 +2835,7 @@ SELECT * FROM test2;
|
||||
|
||||
**Пример**
|
||||
|
||||
Какие изменения привносит включение и выключение настройки:
|
||||
Какие изменения привносит включение и выключение настройки:
|
||||
|
||||
Запрос:
|
||||
|
||||
@ -3023,4 +3045,17 @@ SETTINGS index_granularity = 8192 │
|
||||
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) <!--hide-->
|
||||
## external_table_functions_use_nulls {#external-table-functions-use-nulls}
|
||||
|
||||
Определяет, как табличные функции [mysql](../../sql-reference/table-functions/mysql.md), [postgresql](../../sql-reference/table-functions/postgresql.md) и [odbc](../../sql-reference/table-functions/odbc.md)] используют Nullable столбцы.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- 0 — табличная функция явно использует Nullable столбцы.
|
||||
- 1 — табличная функция неявно использует Nullable столбцы.
|
||||
|
||||
Значение по умолчанию: `1`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Если установлено значение `0`, то табличная функция не делает Nullable столбцы, а вместо NULL выставляет значения по умолчанию для скалярного типа. Это также применимо для значений NULL внутри массивов.
|
||||
|
||||
@ -84,12 +84,10 @@ ClickHouse не удаляет данные из таблица автомати
|
||||
- `forwarded_for` ([String](../../sql-reference/data-types/string.md)) — HTTP заголовок `X-Forwarded-For`.
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — `ключ квоты` из настроек [квот](quotas.md) (см. `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse.
|
||||
- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events
|
||||
- `Settings` ([Map(String, String)](../../sql-reference/data-types/array.md)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1.
|
||||
- `log_comment` ([String](../../sql-reference/data-types/string.md)) — комментарий к записи в логе. Представляет собой произвольную строку, длина которой должна быть не больше, чем [max_query_size](../../operations/settings/settings.md#settings-max_query_size). Если нет комментария, то пустая строка.
|
||||
- `thread_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — идентификаторы потоков, участвующих в обработке запросов.
|
||||
- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events
|
||||
- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики, перечисленные в столбце `ProfileEvents.Names`.
|
||||
- `Settings.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1.
|
||||
- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — значения настроек, которые перечислены в столбце `Settings.Names`.
|
||||
- `used_aggregate_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — канонические имена `агрегатных функций`, использованных при выполнении запроса.
|
||||
- `used_aggregate_function_combinators` ([Array(String)](../../sql-reference/data-types/array.md)) — канонические имена `комбинаторов агрегатных функций`, использованных при выполнении запроса.
|
||||
- `used_database_engines` ([Array(String)](../../sql-reference/data-types/array.md)) — канонические имена `движков баз данных`, использованных при выполнении запроса.
|
||||
@ -109,68 +107,49 @@ SELECT * FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDa
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
type: QueryFinish
|
||||
event_date: 2021-03-18
|
||||
event_time: 2021-03-18 20:54:18
|
||||
event_time_microseconds: 2021-03-18 20:54:18.676686
|
||||
query_start_time: 2021-03-18 20:54:18
|
||||
query_start_time_microseconds: 2021-03-18 20:54:18.673934
|
||||
query_duration_ms: 2
|
||||
read_rows: 100
|
||||
read_bytes: 800
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 2
|
||||
result_bytes: 4858
|
||||
memory_usage: 0
|
||||
current_database: default
|
||||
query: SELECT uniqArray([1, 1, 2]), SUBSTRING('Hello, world', 7, 5), flatten([[[BIT_AND(123)]], [[mod(3, 2)], [CAST('1' AS INTEGER)]]]), week(toDate('2000-12-05')), CAST(arrayJoin([NULL, NULL]) AS Nullable(TEXT)), avgOrDefaultIf(number, number % 2), sumOrNull(number), toTypeName(sumOrNull(number)), countIf(toDate('2000-12-05') + number as d, toDayOfYear(d) % 2) FROM numbers(100)
|
||||
normalized_query_hash: 17858008518552525706
|
||||
query_kind: Select
|
||||
databases: ['_table_function']
|
||||
tables: ['_table_function.numbers']
|
||||
columns: ['_table_function.numbers.number']
|
||||
exception_code: 0
|
||||
type: QueryStart
|
||||
event_date: 2020-09-11
|
||||
event_time: 2020-09-11 10:08:17
|
||||
event_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_start_time: 2020-09-11 10:08:17
|
||||
query_start_time_microseconds: 2020-09-11 10:08:17.063321
|
||||
query_duration_ms: 0
|
||||
read_rows: 0
|
||||
read_bytes: 0
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
result_rows: 0
|
||||
result_bytes: 0
|
||||
memory_usage: 0
|
||||
current_database: default
|
||||
query: INSERT INTO test1 VALUES
|
||||
exception_code: 0
|
||||
exception:
|
||||
stack_trace:
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 58f3d392-0fa0-4663-ae1d-29917a1a9c9c
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 37486
|
||||
initial_user: default
|
||||
initial_query_id: 58f3d392-0fa0-4663-ae1d-29917a1a9c9c
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 37486
|
||||
interface: 1
|
||||
os_user: sevirov
|
||||
client_hostname: clickhouse.ru-central1.internal
|
||||
client_name: ClickHouse
|
||||
client_revision: 54447
|
||||
client_version_major: 21
|
||||
client_version_minor: 4
|
||||
client_version_patch: 1
|
||||
http_method: 0
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
address: ::ffff:127.0.0.1
|
||||
port: 33452
|
||||
initial_user: default
|
||||
initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef
|
||||
initial_address: ::ffff:127.0.0.1
|
||||
initial_port: 33452
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
http_referer:
|
||||
forwarded_for:
|
||||
quota_key:
|
||||
revision: 54449
|
||||
log_comment:
|
||||
thread_ids: [587,11939]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','ArenaAllocChunks','ArenaAllocBytes','FunctionExecute','TableFunctionExecute','NetworkSendElapsedMicroseconds','SelectedRows','SelectedBytes','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SystemTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes']
|
||||
ProfileEvents.Values: [1,1,36,1,10,2,1048680,1,4096,36,1,110,100,800,77,1,3137,1476,1101,8,2577,8192]
|
||||
Settings.Names: ['load_balancing','max_memory_usage']
|
||||
Settings.Values: ['random','10000000000']
|
||||
used_aggregate_functions: ['groupBitAnd','avg','sum','count','uniq']
|
||||
used_aggregate_function_combinators: ['OrDefault','If','OrNull','Array']
|
||||
used_database_engines: []
|
||||
used_data_type_families: ['String','Array','Int32','Nullable']
|
||||
used_dictionaries: []
|
||||
used_formats: []
|
||||
used_functions: ['toWeek','CAST','arrayFlatten','toTypeName','toDayOfYear','addDays','array','toDate','modulo','substring','plus']
|
||||
used_storages: []
|
||||
used_table_functions: ['numbers']
|
||||
revision: 54440
|
||||
thread_ids: []
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
@ -57,8 +57,7 @@ ClickHouse не удаляет данные из таблицы автомати
|
||||
- `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`.
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse.
|
||||
- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events).
|
||||
- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`.
|
||||
- `ProfileEvents` ([Map(String, UInt64)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events).
|
||||
|
||||
**Пример**
|
||||
|
||||
@ -97,17 +96,16 @@ initial_port: 33452
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
revision: 54440
|
||||
ProfileEvents.Names: ['Query','InsertQuery','FileOpen','WriteBufferFromFileDescriptorWrite','WriteBufferFromFileDescriptorWriteBytes','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','FunctionExecute','CreatedWriteBufferOrdinary','DiskWriteElapsedMicroseconds','NetworkReceiveElapsedMicroseconds','NetworkSendElapsedMicroseconds','InsertedRows','InsertedBytes','SelectedRows','SelectedBytes','MergeTreeDataWriterRows','MergeTreeDataWriterUncompressedBytes','MergeTreeDataWriterCompressedBytes','MergeTreeDataWriterBlocks','MergeTreeDataWriterBlocksAlreadySorted','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSReadChars','OSWriteChars']
|
||||
ProfileEvents.Values: [1,1,11,11,591,148,3,71,29,6533808,1,11,72,18,47,1,12,1,12,1,12,189,1,1,10,2,70853,2748,49,2747,45056,422,1520]
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
@ -31,6 +31,8 @@ ClickHouse поддерживает синтаксис `COUNT(DISTINCT ...)`. П
|
||||
|
||||
Запрос `SELECT count() FROM table` не оптимизирован, поскольку количество записей в таблице не хранится отдельно. Он выбирает небольшой столбец из таблицы и подсчитывает количество значений в нём.
|
||||
|
||||
При этом запрос `SELECT count(nullable_column) FROM table` может быть оптимизирован включением настройки [optimize_functions_to_subcolumns](../../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [null](../../../sql-reference/data-types/nullable.md#finding-null) вместо чтения всех данных столбца. Запрос `SELECT count(n) FROM table` преобразуется к запросу `SELECT sum(NOT n.null) FROM table`.
|
||||
|
||||
**Примеры**
|
||||
|
||||
Пример 1:
|
||||
@ -68,4 +70,3 @@ SELECT count(DISTINCT num) FROM t
|
||||
```
|
||||
|
||||
Этот пример показывает, что `count(DISTINCT num)` выполняется с помощью функции `uniqExact` в соответствии со значением настройки `count_distinct_implementation`.
|
||||
|
||||
|
||||
@ -2,7 +2,9 @@
|
||||
toc_priority: 202
|
||||
---
|
||||
|
||||
# quantileExact {#quantileexact}
|
||||
# Функции quantileExact {#quantileexact-functions}
|
||||
|
||||
## quantileExact {#quantileexact}
|
||||
|
||||
Точно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) числовой последовательности.
|
||||
|
||||
@ -23,7 +25,6 @@ quantileExact(level)(expr)
|
||||
- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)).
|
||||
- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Квантиль заданного уровня.
|
||||
@ -50,7 +51,7 @@ SELECT quantileExact(number) FROM numbers(10)
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
# quantileExactLow {#quantileexactlow}
|
||||
## quantileExactLow {#quantileexactlow}
|
||||
|
||||
Как и `quantileExact`, эта функция вычисляет точный [квантиль](https://en.wikipedia.org/wiki/Quantile) числовой последовательности данных.
|
||||
|
||||
@ -67,7 +68,7 @@ SELECT quantileExactLow(0.1)(number) FROM numbers(10)
|
||||
│ 1 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
При использовании в запросе нескольких функций `quantile*` с разными уровнями, внутренние состояния не объединяются (то есть запрос работает менее эффективно). В этом случае используйте функцию [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles).
|
||||
|
||||
**Синтаксис**
|
||||
@ -83,7 +84,6 @@ quantileExact(level)(expr)
|
||||
- `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Квантиль заданного уровня.
|
||||
@ -109,7 +109,7 @@ SELECT quantileExactLow(number) FROM numbers(10)
|
||||
│ 4 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
# quantileExactHigh {#quantileexacthigh}
|
||||
## quantileExactHigh {#quantileexacthigh}
|
||||
|
||||
Как и `quantileExact`, эта функция вычисляет точный [квантиль](https://en.wikipedia.org/wiki/Quantile) числовой последовательности данных.
|
||||
|
||||
@ -134,7 +134,6 @@ quantileExactHigh(level)(expr)
|
||||
- `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median).
|
||||
- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Квантиль заданного уровня.
|
||||
@ -161,8 +160,111 @@ SELECT quantileExactHigh(number) FROM numbers(10)
|
||||
└───────────────────────────┘
|
||||
```
|
||||
|
||||
## quantileExactExclusive {#quantileexactexclusive}
|
||||
|
||||
Точно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) числовой последовательности.
|
||||
|
||||
Чтобы получить точный результат, все переданные значения собираются в массив, который затем частично сортируется. Таким образом, функция потребляет объем памяти `O(n)`, где `n` — количество переданных значений. Для небольшого числа значений эта функция эффективна.
|
||||
|
||||
Эта функция эквивалентна Excel функции [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba), [тип R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample).
|
||||
|
||||
Если в одном запросе вызывается несколько функций `quantileExactExclusive` с разными значениями `level`, эти функции вычисляются независимо друг от друга. В таких случаях используйте функцию [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantilesexactexclusive), запрос будет выполняться эффективнее.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
quantileExactExclusive(level)(expr)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `expr` — выражение, зависящее от значений столбцов. Возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `level` — уровень квантиля. Необязательный параметр. Возможные значения: (0, 1) — граничные значения не учитываются. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Квантиль заданного уровня.
|
||||
|
||||
Тип:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) для входных данных числового типа.
|
||||
- [Date](../../../sql-reference/data-types/date.md), если входные значения имеют тип `Date`.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md), если входные значения имеют тип `DateTime`.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantileExactExclusive(0.6)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─quantileExactExclusive(0.6)(x)─┐
|
||||
│ 599.6 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
## quantileExactInclusive {#quantileexactinclusive}
|
||||
|
||||
Точно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) числовой последовательности.
|
||||
|
||||
Чтобы получить точный результат, все переданные значения собираются в массив, который затем частично сортируется. Таким образом, функция потребляет объем памяти `O(n)`, где `n` — количество переданных значений. Для небольшого числа значений эта функция эффективна.
|
||||
|
||||
Эта функция эквивалентна Excel функции [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed), [тип R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample).
|
||||
|
||||
Если в одном запросе вызывается несколько функций `quantileExactInclusive` с разными значениями `level`, эти функции вычисляются независимо друг от друга. В таких случаях используйте функцию [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantilesexactinclusive), запрос будет выполняться эффективнее.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
quantileExactInclusive(level)(expr)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `expr` — выражение, зависящее от значений столбцов. Возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `level` — уровень квантиля. Необязательный параметр. Возможные значения: [0, 1] — граничные значения учитываются. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Квантиль заданного уровня.
|
||||
|
||||
Тип:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) для входных данных числового типа.
|
||||
- [Date](../../../sql-reference/data-types/date.md), если входные значения имеют тип `Date`.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md), если входные значения имеют тип `DateTime`.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantileExactInclusive(0.6)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─quantileExactInclusive(0.6)(x)─┐
|
||||
│ 599.4 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
- [median](../../../sql-reference/aggregate-functions/reference/median.md#median)
|
||||
- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles)
|
||||
|
||||
|
||||
@ -2,9 +2,114 @@
|
||||
toc_priority: 201
|
||||
---
|
||||
|
||||
# quantiles {#quantiles}
|
||||
# Функции для нескольких квантилей {#quantiles-functions}
|
||||
|
||||
Syntax: `quantiles(level1, level2, …)(x)`
|
||||
## quantiles {#quantiles}
|
||||
|
||||
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
||||
Синтаксис: `quantiles(level1, level2, …)(x)`
|
||||
|
||||
Все функции для вычисления квантилей имеют соответствующие функции для вычисления нескольких квантилей: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. Эти функции вычисляют все квантили указанных уровней в один проход и возвращают массив с вычисленными значениями.
|
||||
|
||||
## quantilesExactExclusive {#quantilesexactexclusive}
|
||||
|
||||
Точно вычисляет [квантили](https://ru.wikipedia.org/wiki/Квантиль) числовой последовательности.
|
||||
|
||||
Чтобы получить точный результат, все переданные значения собираются в массив, который затем частично сортируется. Таким образом, функция потребляет объем памяти `O(n)`, где `n` — количество переданных значений. Для небольшого числа значений эта функция эффективна.
|
||||
|
||||
Эта функция эквивалентна Excel функции [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba), [тип R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample).
|
||||
|
||||
С наборами уровней работает эффективнее, чем [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive).
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
quantilesExactExclusive(level1, level2, ...)(expr)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `expr` — выражение, зависящее от значений столбцов. Возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `level` — уровень квантилей. Возможные значения: (0, 1) — граничные значения не учитываются. [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Возвращаемые значения**
|
||||
|
||||
- [Массив](../../../sql-reference/data-types/array.md) квантилей указанных уровней.
|
||||
|
||||
Тип значений массива:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) для входных данных числового типа.
|
||||
- [Date](../../../sql-reference/data-types/date.md), если входные значения имеют тип `Date`.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md), если входные значения имеют тип `DateTime`.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantilesExactExclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─quantilesExactExclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x)─┐
|
||||
│ [249.25,499.5,749.75,899.9,949.9499999999999,989.99,998.999] │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## quantilesExactInclusive {#quantilesexactinclusive}
|
||||
|
||||
Точно вычисляет [квантили](https://ru.wikipedia.org/wiki/Квантиль) числовой последовательности.
|
||||
|
||||
Чтобы получить точный результат, все переданные значения собираются в массив, который затем частично сортируется. Таким образом, функция потребляет объем памяти `O(n)`, где `n` — количество переданных значений. Для небольшого числа значений эта функция эффективна.
|
||||
|
||||
Эта функция эквивалентна Excel функции [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed), [тип R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample).
|
||||
|
||||
С наборами уровней работает эффективнее, чем [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantilesexactinclusive).
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
quantilesExactInclusive(level1, level2, ...)(expr)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `expr` — выражение, зависящее от значений столбцов. Возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `level` — уровень квантилей. Возможные значения: [0, 1] — граничные значения учитываются. [Float](../../../sql-reference/data-types/float.md).
|
||||
|
||||
**Возвращаемые значения**
|
||||
|
||||
- [Массив](../../../sql-reference/data-types/array.md) квантилей указанных уровней.
|
||||
|
||||
Тип значений массива:
|
||||
|
||||
- [Float64](../../../sql-reference/data-types/float.md) для входных данных числового типа.
|
||||
- [Date](../../../sql-reference/data-types/date.md), если входные значения имеют тип `Date`.
|
||||
- [DateTime](../../../sql-reference/data-types/datetime.md), если входные значения имеют тип `DateTime`.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE num AS numbers(1000);
|
||||
|
||||
SELECT quantilesExactInclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x) FROM (SELECT number AS x FROM num);
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─quantilesExactInclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x)─┐
|
||||
│ [249.75,499.5,749.25,899.1,949.05,989.01,998.001] │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -8,6 +8,7 @@ toc_title: Map(key, value)
|
||||
Тип данных `Map(key, value)` хранит пары `ключ:значение`.
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md).
|
||||
- `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
@ -61,6 +62,36 @@ SELECT a['key3'] FROM table_map;
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
## Подстолбцы Map.keys и Map.values {#map-subcolumns}
|
||||
|
||||
Для оптимизации обработки столбцов `Map` в некоторых случаях можно использовать подстолбцы `keys` и `values` вместо чтения всего столбца.
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE t_map (`a` Map(String, UInt64)) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_map VALUES (map('key1', 1, 'key2', 2, 'key3', 3));
|
||||
|
||||
SELECT a.keys FROM t_map;
|
||||
|
||||
SELECT a.values FROM t_map;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─a.keys─────────────────┐
|
||||
│ ['key1','key2','key3'] │
|
||||
└────────────────────────┘
|
||||
|
||||
┌─a.values─┐
|
||||
│ [1,2,3] │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
**См. также**
|
||||
|
||||
- функция [map()](../../sql-reference/functions/tuple-map-functions.md#function-map)
|
||||
|
||||
@ -51,13 +51,13 @@ LIFETIME(300)
|
||||
LIFETIME(MIN 300 MAX 360)
|
||||
```
|
||||
|
||||
Если `<min>0</min>` и `<max>0</max>`, ClickHouse не перегружает словарь по истечению времени.
|
||||
В этм случае, ClickHouse может перезагрузить данные словаря если изменился XML файл с конфигурацией словаря или если была выполнена команда `SYSTEM RELOAD DICTIONARY`.
|
||||
Если `<min>0</min>` и `<max>0</max>`, ClickHouse не перезагружает словарь по истечении времени.
|
||||
В этом случае ClickHouse может перезагрузить данные словаря, если изменился XML файл с конфигурацией словаря или если была выполнена команда `SYSTEM RELOAD DICTIONARY`.
|
||||
|
||||
При обновлении словарей сервер ClickHouse применяет различную логику в зависимости от типа [источника](external-dicts-dict-sources.md):
|
||||
|
||||
- У текстового файла проверяется время модификации. Если время изменилось по отношению к запомненному ранее, то словарь обновляется.
|
||||
- Для MySQL источника, время модификации проверяется запросом `SHOW TABLE STATUS` (для MySQL 8 необходимо отключить кеширование мета-информации в MySQL `set global information_schema_stats_expiry=0`.
|
||||
- Для MySQL источника время модификации проверяется запросом `SHOW TABLE STATUS` (для MySQL 8 необходимо отключить кеширование мета-информации в MySQL `set global information_schema_stats_expiry=0`).
|
||||
- Словари из других источников по умолчанию обновляются каждый раз.
|
||||
|
||||
Для других источников (ODBC, PostgreSQL, ClickHouse и т.д.) можно настроить запрос, который позволит обновлять словари только в случае их фактического изменения, а не каждый раз. Чтобы это сделать необходимо выполнить следующие условия/действия:
|
||||
@ -86,4 +86,34 @@ SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source wher
|
||||
...
|
||||
```
|
||||
|
||||
Для словарей `Cache`, `ComplexKeyCache`, `SSDCache` и `SSDComplexKeyCache` поддерживается как синхронное, так и асинхронное обновление.
|
||||
Для словарей `Cache`, `ComplexKeyCache`, `SSDCache` и `SSDComplexKeyCache` поддерживается как синхронное, так и асинхронное обновление.
|
||||
|
||||
Словари `Flat`, `Hashed` и `ComplexKeyHashed` могут запрашивать только те данные, которые были изменены после предыдущего обновления. Если `update_field` указано как часть конфигурации источника словаря, к запросу данных будет добавлено время предыдущего обновления в секундах. В зависимости от типа источника (Executable, HTTP, MySQL, PostgreSQL, ClickHouse, ODBC) к `update_field` будет применена соответствующая логика перед запросом данных из внешнего источника.
|
||||
|
||||
- Если источник HTTP, то `update_field` будет добавлено в качестве параметра запроса, а время последнего обновления — в качестве значения параметра.
|
||||
- Если источник Executable, то `update_field` будет добавлено в качестве аргумента исполняемого скрипта, время последнего обновления — в качестве значения аргумента.
|
||||
- Если источник ClickHouse, MySQL, PostgreSQL или ODBC, то будет дополнительная часть запроса `WHERE`, где `update_field` будет больше или равно времени последнего обновления.
|
||||
|
||||
Если установлена опция `update_field`, то может быть установлена дополнительная опция `update_lag`. Значение `update_lag` вычитается из времени предыдущего обновления перед запросом обновленных данных.
|
||||
|
||||
Пример настройки:
|
||||
|
||||
``` xml
|
||||
<dictionary>
|
||||
...
|
||||
<clickhouse>
|
||||
...
|
||||
<update_field>added_time</update_field>
|
||||
<update_lag>15</update_lag>
|
||||
</clickhouse>
|
||||
...
|
||||
</dictionary>
|
||||
```
|
||||
|
||||
или
|
||||
|
||||
``` sql
|
||||
...
|
||||
SOURCE(CLICKHOUSE(... update_field 'added_time' update_lag 15))
|
||||
...
|
||||
```
|
||||
|
||||
@ -11,18 +11,24 @@ toc_title: "Массивы"
|
||||
Тип результата - UInt8.
|
||||
Функция также работает для строк.
|
||||
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [size0](../../sql-reference/data-types/array.md#array-size) вместо чтения и обработки всего столбца массива. Запрос `SELECT empty(arr) FROM table` преобразуется к запросу `SELECT arr.size0 = 0 FROM TABLE`.
|
||||
|
||||
## notEmpty {#function-notempty}
|
||||
|
||||
Возвращает 0 для пустого массива, и 1 для непустого массива.
|
||||
Тип результата - UInt8.
|
||||
Функция также работает для строк.
|
||||
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [size0](../../sql-reference/data-types/array.md#array-size) вместо чтения и обработки всего столбца массива. Запрос `SELECT notEmpty(arr) FROM table` преобразуется к запросу `SELECT arr.size0 != 0 FROM TABLE`.
|
||||
|
||||
## length {#array_functions-length}
|
||||
|
||||
Возвращает количество элементов в массиве.
|
||||
Тип результата - UInt64.
|
||||
Функция также работает для строк.
|
||||
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [size0](../../sql-reference/data-types/array.md#array-size) вместо чтения и обработки всего столбца массива. Запрос `SELECT length(arr) FROM table` преобразуется к запросу `SELECT arr.size0 FROM TABLE`.
|
||||
|
||||
## emptyArrayUInt8, emptyArrayUInt16, emptyArrayUInt32, emptyArrayUInt64 {#emptyarrayuint8-emptyarrayuint16-emptyarrayuint32-emptyarrayuint64}
|
||||
|
||||
## emptyArrayInt8, emptyArrayInt16, emptyArrayInt32, emptyArrayInt64 {#emptyarrayint8-emptyarrayint16-emptyarrayint32-emptyarrayint64}
|
||||
|
||||
@ -224,6 +224,8 @@ SELECT mapContains(a, 'name') FROM test;
|
||||
|
||||
Возвращает все ключи контейнера `map`.
|
||||
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [keys](../../sql-reference/data-types/map.md#map-subcolumns) вместо чтения и обработки данных всего столбца. Запрос `SELECT mapKeys(m) FROM table` преобразуется к запросу `SELECT m.keys FROM table`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
@ -265,6 +267,8 @@ SELECT mapKeys(a) FROM test;
|
||||
|
||||
Возвращает все значения контейнера `map`.
|
||||
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [values](../../sql-reference/data-types/map.md#map-subcolumns) вместо чтения и обработки данных всего столбца. Запрос `SELECT mapValues(m) FROM table` преобразуется к запросу `SELECT m.values FROM table`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
```sql
|
||||
|
||||
@ -283,6 +283,8 @@ ClickHouse поддерживает операторы `IS NULL` и `IS NOT NULL
|
||||
- `0` в обратном случае.
|
||||
- Для прочих значений оператор `IS NULL` всегда возвращает `0`.
|
||||
|
||||
Оператор можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` читается только подстолбец [null](../../sql-reference/data-types/nullable.md#finding-null) вместо чтения и обработки данных всего столбца. Запрос `SELECT n IS NULL FROM table` преобразуется к запросу `SELECT n.null FROM TABLE`.
|
||||
|
||||
<!-- -->
|
||||
|
||||
``` sql
|
||||
@ -302,6 +304,8 @@ SELECT x+100 FROM t_null WHERE y IS NULL
|
||||
- `1`, в обратном случае.
|
||||
- Для прочих значений оператор `IS NOT NULL` всегда возвращает `1`.
|
||||
|
||||
Оператор можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` читается только подстолбец [null](../../sql-reference/data-types/nullable.md#finding-null) вместо чтения и обработки данных всего столбца. Запрос `SELECT n IS NOT NULL FROM table` преобразуется к запросу `SELECT NOT n.null FROM TABLE`.
|
||||
|
||||
<!-- -->
|
||||
|
||||
``` sql
|
||||
|
||||
@ -19,6 +19,8 @@ toc_title: PARTITION
|
||||
- [UNFREEZE PARTITION](#alter_unfreeze-partition) — удалить резервную копию партиции;
|
||||
- [FETCH PARTITION](#alter_fetch-partition) — скачать партицию с другого сервера;
|
||||
- [MOVE PARTITION\|PART](#alter_move-partition) — переместить партицию/кускок на другой диск или том.
|
||||
- [UPDATE IN PARTITION](#update-in-partition) — обновить данные внутри партиции по условию.
|
||||
- [DELETE IN PARTITION](#delete-in-partition) — удалить данные внутри партиции по условию.
|
||||
|
||||
## DETACH PARTITION\|PART {#alter_detach-partition}
|
||||
|
||||
|
||||
@ -5,14 +5,14 @@ toc_title: SYSTEM
|
||||
|
||||
# Запросы SYSTEM {#query-language-system}
|
||||
|
||||
- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries)
|
||||
- [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries)
|
||||
- [RELOAD DICTIONARIES](#query_language-system-reload-dictionaries)
|
||||
- [RELOAD DICTIONARY](#query_language-system-reload-dictionary)
|
||||
- [RELOAD MODELS](#query_language-system-reload-models)
|
||||
- [RELOAD MODEL](#query_language-system-reload-model)
|
||||
- [DROP DNS CACHE](#query_language-system-drop-dns-cache)
|
||||
- [DROP MARK CACHE](#query_language-system-drop-mark-cache)
|
||||
- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache)
|
||||
- [DROP UNCOMPRESSED CACHE](#query_language-system-drop-uncompressed-cache)
|
||||
- [DROP COMPILED EXPRESSION CACHE](#query_language-system-drop-compiled-expression-cache)
|
||||
- [DROP REPLICA](#query_language-system-drop-replica)
|
||||
- [FLUSH LOGS](#query_language-system-flush_logs)
|
||||
@ -24,10 +24,10 @@ toc_title: SYSTEM
|
||||
- [START DISTRIBUTED SENDS](#query_language-system-start-distributed-sends)
|
||||
- [STOP MERGES](#query_language-system-stop-merges)
|
||||
- [START MERGES](#query_language-system-start-merges)
|
||||
- [STOP TTL MERGES](#query_language-stop-ttl-merges)
|
||||
- [START TTL MERGES](#query_language-start-ttl-merges)
|
||||
- [STOP MOVES](#query_language-stop-moves)
|
||||
- [START MOVES](#query_language-start-moves)
|
||||
- [STOP TTL MERGES](#query_language-stop-ttl-merges)
|
||||
- [START TTL MERGES](#query_language-start-ttl-merges)
|
||||
- [STOP MOVES](#query_language-stop-moves)
|
||||
- [START MOVES](#query_language-start-moves)
|
||||
- [STOP FETCHES](#query_language-system-stop-fetches)
|
||||
- [START FETCHES](#query_language-system-start-fetches)
|
||||
- [STOP REPLICATED SENDS](#query_language-system-start-replicated-sends)
|
||||
@ -36,13 +36,13 @@ toc_title: SYSTEM
|
||||
- [START REPLICATION QUEUES](#query_language-system-start-replication-queues)
|
||||
- [SYNC REPLICA](#query_language-system-sync-replica)
|
||||
- [RESTART REPLICA](#query_language-system-restart-replica)
|
||||
- [RESTART REPLICAS](#query_language-system-restart-replicas)
|
||||
- [RESTART REPLICAS](#query_language-system-restart-replicas)
|
||||
|
||||
## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries}
|
||||
## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries}
|
||||
Перегружает все [Встроенные словари](../dictionaries/internal-dicts.md).
|
||||
По умолчанию встроенные словари выключены.
|
||||
По умолчанию встроенные словари выключены.
|
||||
Всегда возвращает `Ok.`, вне зависимости от результата обновления встроенных словарей.
|
||||
|
||||
|
||||
## RELOAD DICTIONARIES {#query_language-system-reload-dictionaries}
|
||||
|
||||
Перегружает все словари, которые были успешно загружены до этого.
|
||||
@ -115,7 +115,7 @@ SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk';
|
||||
|
||||
## DROP COMPILED EXPRESSION CACHE {#query_language-system-drop-compiled-expression-cache}
|
||||
Сбрасывает кеш скомпилированных выражений. Используется при разработке ClickHouse и тестах производительности.
|
||||
Компилированные выражения используются когда включена настройка уровня запрос/пользователь/профиль [compile](../../operations/settings/settings.md#compile)
|
||||
Cкомпилированные выражения используются когда включена настройка уровня запрос/пользователь/профиль [compile-expressions](../../operations/settings/settings.md#compile-expressions)
|
||||
|
||||
## FLUSH LOGS {#query_language-system-flush_logs}
|
||||
|
||||
@ -194,7 +194,7 @@ SYSTEM START MERGES [ON VOLUME <volume_name> | [db.]merge_tree_family_table_name
|
||||
SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
### START TTL MERGES {#query_language-start-ttl-merges}
|
||||
### START TTL MERGES {#query_language-start-ttl-merges}
|
||||
|
||||
Запускает фоновые процессы удаления старых данных основанные на [выражениях TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) для таблиц семейства MergeTree:
|
||||
Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
|
||||
@ -203,7 +203,7 @@ SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
### STOP MOVES {#query_language-stop-moves}
|
||||
### STOP MOVES {#query_language-stop-moves}
|
||||
|
||||
Позволяет остановить фоновые процессы переноса данных основанные [табличных выражениях TTL с использованием TO VOLUME или TO DISK](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
|
||||
Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
|
||||
@ -212,7 +212,7 @@ SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
### START MOVES {#query_language-start-moves}
|
||||
### START MOVES {#query_language-start-moves}
|
||||
|
||||
Запускает фоновые процессы переноса данных основанные [табличных выражениях TTL с использованием TO VOLUME или TO DISK](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
|
||||
Возвращает `Ok.` даже если указана несуществующая таблица или таблица имеет тип отличный от MergeTree. Возвращает ошибку если указана не существующая база данных:
|
||||
@ -261,7 +261,7 @@ SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### STOP REPLICATION QUEUES {#query_language-system-stop-replication-queues}
|
||||
|
||||
Останавливает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
|
||||
Останавливает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
|
||||
|
||||
``` sql
|
||||
SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
|
||||
@ -269,7 +269,7 @@ SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### START REPLICATION QUEUES {#query_language-system-start-replication-queues}
|
||||
|
||||
Запускает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
|
||||
Запускает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
|
||||
|
||||
``` sql
|
||||
SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
|
||||
@ -277,7 +277,7 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### SYNC REPLICA {#query_language-system-sync-replica}
|
||||
|
||||
Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, будет работать до достижения `receive_timeout`, если синхронизация для таблицы отключена в настоящий момент времени:
|
||||
Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, будет работать до достижения `receive_timeout`, если синхронизация для таблицы отключена в настоящий момент времени:
|
||||
|
||||
``` sql
|
||||
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
|
||||
|
||||
@ -38,6 +38,18 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_
|
||||
|
||||
Остальные условия и ограничение выборки `LIMIT` будут выполнены в ClickHouse только после выполнения запроса к MySQL.
|
||||
|
||||
Поддерживает несколько реплик, которые должны быть перечислены через `|`. Например:
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql{1|2|3}:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
или
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
Объект таблицы с теми же столбцами, что и в исходной таблице MySQL.
|
||||
|
||||
@ -43,6 +43,18 @@ PostgreSQL массивы конвертируются в массивы ClickHo
|
||||
|
||||
!!! info "Примечание"
|
||||
Будьте внимательны, в PostgreSQL массивы, созданные как `type_name[]`, являются многомерными и могут содержать в себе разное количество измерений в разных строках одной таблицы. Внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом измерений во всех строках таблицы.
|
||||
|
||||
Поддерживает несколько реплик, которые должны быть перечислены через `|`. Например:
|
||||
|
||||
```sql
|
||||
SELECT name FROM postgresql(`postgres{1|2|3}:5432`, 'postgres_database', 'postgres_table', 'user', 'password');
|
||||
```
|
||||
|
||||
или
|
||||
|
||||
```sql
|
||||
SELECT name FROM postgresql(`postgres1:5431|postgres2:5432`, 'postgres_database', 'postgres_table', 'user', 'password');
|
||||
```
|
||||
|
||||
При использовании словаря PostgreSQL поддерживается приоритет реплик. Чем больше номер реплики, тем ниже ее приоритет. Наивысший приоритет у реплики с номером `0`.
|
||||
|
||||
|
||||
48
docs/ru/sql-reference/table-functions/s3Cluster.md
Normal file
48
docs/ru/sql-reference/table-functions/s3Cluster.md
Normal file
@ -0,0 +1,48 @@
|
||||
---
|
||||
toc_priority: 55
|
||||
toc_title: s3Cluster
|
||||
---
|
||||
|
||||
# Табличная функция s3Cluster {#s3Cluster-table-function}
|
||||
|
||||
Позволяет обрабатывать файлы из [Amazon S3](https://aws.amazon.com/s3/) параллельно из многих узлов в указанном кластере. На узле-инициаторе функция создает соединение со всеми узлами в кластере, заменяет символы '*' в пути к файлу S3 и динамически отправляет каждый файл. На рабочем узле функция запрашивает у инициатора следующую задачу и обрабатывает ее. Это повторяется до тех пор, пока все задачи не будут завершены.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `cluster_name` — имя кластера, используемое для создания набора адресов и параметров подключения к удаленным и локальным серверам.
|
||||
- `source` — URL файла или нескольких файлов. Поддерживает следующие символы подстановки: `*`, `?`, `{'abc','def'}` и `{N..M}`, где `N`, `M` — числа, `abc`, `def` — строки. Подробнее смотрите в разделе [Символы подстановки](../../engines/table-engines/integrations/s3.md#wildcards-in-path).
|
||||
- `access_key_id` и `secret_access_key` — ключи, указывающие на учетные данные для использования с точкой приема запроса. Необязательные параметры.
|
||||
- `format` — [формат](../../interfaces/formats.md#formats) файла.
|
||||
- `structure` — структура таблицы. Формат `'column1_name column1_type, column2_name column2_type, ...'`.
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
Таблица с указанной структурой для чтения или записи данных в указанный файл.
|
||||
|
||||
**Примеры**
|
||||
|
||||
Вывод данных из всех файлов кластера `cluster_simple`:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM s3Cluster('cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon);
|
||||
```
|
||||
|
||||
Подсчет общего количества строк во всех файлах кластера `cluster_simple`:
|
||||
|
||||
``` sql
|
||||
SELECT count(*) FROM s3Cluster('cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))');
|
||||
```
|
||||
|
||||
!!! warning "Внимание"
|
||||
Если список файлов содержит диапазоны чисел с ведущими нулями, используйте конструкцию с фигурными скобками для каждой цифры отдельно или используйте `?`.
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
- [Движок таблиц S3](../../engines/table-engines/integrations/s3.md)
|
||||
- [Табличная функция s3](../../sql-reference/table-functions/s3.md)
|
||||
@ -154,9 +154,6 @@ def build(args):
|
||||
if not args.skip_website:
|
||||
website.build_website(args)
|
||||
|
||||
if not args.skip_test_templates:
|
||||
test.test_templates(args.website_dir)
|
||||
|
||||
if not args.skip_docs:
|
||||
generate_cmake_flags_files()
|
||||
|
||||
@ -197,7 +194,6 @@ if __name__ == '__main__':
|
||||
arg_parser.add_argument('--skip-blog', action='store_true')
|
||||
arg_parser.add_argument('--skip-git-log', action='store_true')
|
||||
arg_parser.add_argument('--skip-docs', action='store_true')
|
||||
arg_parser.add_argument('--skip-test-templates', action='store_true')
|
||||
arg_parser.add_argument('--test-only', action='store_true')
|
||||
arg_parser.add_argument('--minify', action='store_true')
|
||||
arg_parser.add_argument('--htmlproofer', action='store_true')
|
||||
|
||||
@ -7,36 +7,6 @@ import bs4
|
||||
import subprocess
|
||||
|
||||
|
||||
def test_template(template_path):
|
||||
if template_path.endswith('amp.html'):
|
||||
# Inline CSS/JS is ok for AMP pages
|
||||
return
|
||||
|
||||
logging.debug(f'Running tests for {template_path} template')
|
||||
with open(template_path, 'r') as f:
|
||||
soup = bs4.BeautifulSoup(
|
||||
f,
|
||||
features='html.parser'
|
||||
)
|
||||
for tag in soup.find_all():
|
||||
style_attr = tag.attrs.get('style')
|
||||
assert not style_attr, f'Inline CSS is prohibited, found {style_attr} in {template_path}'
|
||||
|
||||
if tag.name == 'script':
|
||||
if tag.attrs.get('type') == 'application/ld+json':
|
||||
continue
|
||||
for content in tag.contents:
|
||||
assert not content, f'Inline JavaScript is prohibited, found "{content}" in {template_path}'
|
||||
|
||||
|
||||
def test_templates(base_dir):
|
||||
logging.info('Running tests for templates')
|
||||
for root, _, filenames in os.walk(base_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith('.html'):
|
||||
test_template(os.path.join(root, filename))
|
||||
|
||||
|
||||
def test_single_page(input_path, lang):
|
||||
with open(input_path) as f:
|
||||
soup = bs4.BeautifulSoup(
|
||||
|
||||
@ -54,7 +54,7 @@ SELECT * FROM file_engine_table
|
||||
|
||||
## 在 Clickhouse-local 中的使用 {#zai-clickhouse-local-zhong-de-shi-yong}
|
||||
|
||||
使用 [ﾂ环板-ｮﾂ嘉ｯﾂ偲](../../../engines/table-engines/special/file.md) 时，File 引擎除了 `Format` 之外，还可以接受文件路径参数。可以使用数字或人类可读的名称来指定标准输入/输出流，例如 `0` 或 `stdin`，`1` 或 `stdout`。
|
||||
使用 [clickhouse-local](../../../operations/utilities/clickhouse-local.md) 时，File 引擎除了 `Format` 之外，还可以接收文件路径参数。可以使用数字或名称来指定标准输入/输出流，例如 `0` 或 `stdin`，`1` 或 `stdout`。
|
||||
**例如：**
|
||||
|
||||
``` bash
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# 设置 {#set}
|
||||
# 集合 {#set}
|
||||
|
||||
始终存在于 RAM 中的数据集。它适用于IN运算符的右侧（请参见 «IN运算符» 部分）。
|
||||
|
||||
|
||||
@ -488,7 +488,7 @@ max_alter_threads 2
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<http_handlers>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
``` bash
|
||||
|
||||
@ -796,7 +796,7 @@ ClickHouse使用ZooKeeper存储复制表副本的元数据。 如果未使用复
|
||||
The `index` attribute specifies the node order when trying to connect to the ZooKeeper cluster.
|
||||
|
||||
- `session_timeout` — Maximum timeout for the client session in milliseconds.
|
||||
- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) 隆隆隆隆路虏脢..陇.貌.垄拢卢虏禄.陇.貌路.隆拢脳枚脢虏.麓脢for脱 可选。
|
||||
- `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes)被用作根由ClickHouse服务器使用znodes 可选。
|
||||
- `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional.
|
||||
|
||||
**配置示例**
|
||||
@ -823,7 +823,7 @@ ClickHouse使用ZooKeeper存储复制表副本的元数据。 如果未使用复
|
||||
**另请参阅**
|
||||
|
||||
- [复制](../../engines/table-engines/mergetree-family/replication.md)
|
||||
- [动物园管理员程序员指南](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html)
|
||||
- [zookeeper管理指南](http://zookeeper.apache.org/doc/current/zookeeperProgrammers.html)
|
||||
|
||||
## use_minimalistic_part_header_in_zookeeper {#server-settings-use_minimalistic_part_header_in_zookeeper}
|
||||
|
||||
|
||||
@ -817,21 +817,22 @@ load_balancing = first_or_random
|
||||
为了保持一致性（以获取相同数据拆分的不同部分），此选项仅在设置了采样键时有效。
|
||||
副本滞后不受控制。
|
||||
|
||||
## 编译 {#compile}
|
||||
## compile_expressions {#compile-expressions}
|
||||
|
||||
启用查询的编译。 默认情况下，0（禁用）。
|
||||
啟用或禁用在運行時使用 LLVM 將常用的簡單函數和運算符編譯為本機代碼。
|
||||
|
||||
编译仅用于查询处理管道的一部分：用于聚合的第一阶段（GROUP BY）。
|
||||
如果编译了管道的这一部分，则由于部署周期较短和内联聚合函数调用，查询可能运行得更快。 对于具有多个简单聚合函数的查询，可以看到最大的性能改进（在极少数情况下可快四倍）。 通常，性能增益是微不足道的。 在极少数情况下，它可能会减慢查询执行速度。
|
||||
可能的值：
|
||||
|
||||
## min_count_to_compile {#min-count-to-compile}
|
||||
- 0 — 禁用。
|
||||
- 1 — 啟用。
|
||||
|
||||
在运行编译之前可能使用已编译代码块的次数。 默认情况下，3。
|
||||
For testing, the value can be set to 0: compilation runs synchronously and the query waits for the end of the compilation process before continuing execution. For all other cases, use values starting with 1. Compilation normally takes about 5-10 seconds.
|
||||
如果该值为1或更大，则编译在单独的线程中异步进行。 结果将在准备就绪后立即使用，包括当前正在运行的查询。
|
||||
默認值：`1`。
|
||||
|
||||
对于查询中使用的聚合函数的每个不同组合以及GROUP BY子句中的键类型，都需要编译代码。
|
||||
The results of the compilation are saved in the build directory in the form of .so files. There is no restriction on the number of compilation results since they don't use very much space. Old results will be used after server restarts, except in the case of a server upgrade – in this case, the old results are deleted.
|
||||
## min_count_to_compile_expression {#min-count-to-compile-expression}
|
||||
|
||||
在編譯之前執行相同表達式的最小計數。
|
||||
|
||||
默認值：`3`。
|
||||
|
||||
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
|
||||
**示例**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.data_type_families WHERE alias_to = 'String'
|
||||
SELECT * FROM system.data_type_families WHERE alias_to = 'String';
|
||||
```
|
||||
|
||||
``` text
|
||||
|
||||
@ -76,14 +76,14 @@ ClickHouse不会自动从表中删除数据。更多详情请看 [introduction](
|
||||
- `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的Patch component。
|
||||
- `http_method` (UInt8) — 发起查询的HTTP方法. 可能值:
|
||||
- 0 — TCP接口的查询.
|
||||
- 1 — `GET`
|
||||
- 1 — `GET`
|
||||
- 2 — `POST`
|
||||
- `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — The `UserAgent` The UserAgent header passed in the HTTP request。
|
||||
- `quota_key` ([String](../../sql-reference/data-types/string.md)) — 在[quotas](../../operations/quotas.md) 配置里设置的“quota key” （见 `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
|
||||
- `thread_numbers` ([Array(UInt32)](../../sql-reference/data-types/array.md)) — 参与查询的线程数.
|
||||
- `ProfileEvents.Names` ([Array（String)](../../sql-reference/data-types/array.md)) — 衡量不同指标的计数器。 可以在[system.events](../../operations/system-tables/events.md#system_tables-events)中找到它们的描述。
|
||||
- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — `ProfileEvents.Names` 列中列出的指标的值。
|
||||
- `ProfileEvents` ([Map(String, UInt64))](../../sql-reference/data-types/array.md)) — Counters that measure different metrics. The description of them could be found in the table [系统。活动](../../operations/system-tables/events.md#system_tables-events)
|
||||
- `Settings` ([Map(String, String)](../../sql-reference/data-types/array.md)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` 参数为1。
|
||||
- `thread_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — 参与查询的线程数.
|
||||
- `Settings.Names` ([Array（String)](../../sql-reference/data-types/array.md)) — 客户端运行查询时更改的设置的名称。 要启用对设置的日志记录更改，请将log_query_settings参数设置为1。
|
||||
- `Settings.Values` ([Array（String)](../../sql-reference/data-types/array.md)) — `Settings.Names` 列中列出的设置的值。
|
||||
**示例**
|
||||
@ -133,10 +133,8 @@ http_user_agent:
|
||||
quota_key:
|
||||
revision: 54434
|
||||
thread_ids: []
|
||||
ProfileEvents.Names: []
|
||||
ProfileEvents.Values: []
|
||||
Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage']
|
||||
Settings.Values: ['0','random','1','10000000000']
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
Settings: {'background_pool_size':'32','load_balancing':'random','allow_suspicious_low_cardinality_types':'1','distributed_aggregation_memory_efficient':'1','skip_unavailable_shards':'1','log_queries':'1','max_bytes_before_external_group_by':'20000000000','max_bytes_before_external_sort':'20000000000','allow_introspection_functions':'1'}
|
||||
```
|
||||
|
||||
**另请参阅**
|
||||
|
||||
@ -61,8 +61,7 @@ ClickHouse不会自动从表中删除数据。 看 [导言](../../operations/sys
|
||||
- `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — The `UserAgent` http请求中传递的标头。
|
||||
- `quota_key` ([字符串](../../sql-reference/data-types/string.md)) — The “quota key” 在指定 [配额](../../operations/quotas.md) 设置（见 `keyed`).
|
||||
- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
|
||||
- `ProfileEvents.Names` ([数组（字符串)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [系统。活动](#system_tables-events).
|
||||
- `ProfileEvents.Values` ([数组(UInt64)](../../sql-reference/data-types/array.md)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` 列。
|
||||
- `ProfileEvents` ([数组（字符串, UInt64)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [系统。活动](#system_tables-events).
|
||||
|
||||
**示例**
|
||||
|
||||
@ -108,8 +107,7 @@ http_method: 0
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
revision: 54434
|
||||
ProfileEvents.Names: ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds']
|
||||
ProfileEvents.Values: [1,97,81,5,81]
|
||||
ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'CompressedReadBufferBlocks':1,'CompressedReadBufferBytes':10,'IOBufferAllocs':1,'IOBufferAllocBytes':89,'ContextLock':15,'RWLockAcquiredReadLocks':1}
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@ -95,7 +95,7 @@ SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk';
|
||||
## DROP COMPILED EXPRESSION CACHE {#query_language-system-drop-compiled-expression-cache}
|
||||
|
||||
重置已编译的表达式缓存。用于ClickHouse开发和性能测试。
|
||||
当 `query/user/profile` 启用配置项 [compile](../../operations/settings/settings.md#compile)时，编译的表达式缓存开启。
|
||||
当 `query/user/profile` 启用配置项 [compile-expressions](../../operations/settings/settings.md#compile-expressions)时，编译的表达式缓存开启。
|
||||
|
||||
## FLUSH LOGS {#query_language-system-flush_logs}
|
||||
|
||||
@ -209,7 +209,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
|
||||
|
||||
### STOP FETCHES {#query_language-system-stop-fetches}
|
||||
|
||||
停止后台获取 `ReplicatedMergeTree`系列引擎表中插入的数据块。
|
||||
停止后台获取 `ReplicatedMergeTree`系列引擎表中插入的数据块。
|
||||
不管表引擎类型如何或表/数据库是否存，都返回 `OK.`。
|
||||
|
||||
``` sql
|
||||
@ -218,7 +218,7 @@ SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### START FETCHES {#query_language-system-start-fetches}
|
||||
|
||||
启动后台获取 `ReplicatedMergeTree`系列引擎表中插入的数据块。
|
||||
启动后台获取 `ReplicatedMergeTree`系列引擎表中插入的数据块。
|
||||
不管表引擎类型如何或表/数据库是否存，都返回 `OK.`。
|
||||
|
||||
``` sql
|
||||
@ -227,7 +227,7 @@ SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### STOP REPLICATED SENDS {#query_language-system-start-replicated-sends}
|
||||
|
||||
停止通过后台分发 `ReplicatedMergeTree`系列引擎表中新插入的数据块到集群的其它副本节点。
|
||||
停止通过后台分发 `ReplicatedMergeTree`系列引擎表中新插入的数据块到集群的其它副本节点。
|
||||
|
||||
``` sql
|
||||
SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
|
||||
@ -235,7 +235,7 @@ SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
### START REPLICATED SENDS {#query_language-system-start-replicated-sends}
|
||||
|
||||
启动通过后台分发 `ReplicatedMergeTree`系列引擎表中新插入的数据块到集群的其它副本节点。
|
||||
启动通过后台分发 `ReplicatedMergeTree`系列引擎表中新插入的数据块到集群的其它副本节点。
|
||||
|
||||
``` sql
|
||||
SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
|
||||
|
||||
120
docs/zh/sql-reference/table-functions/postgresql.md
Normal file
120
docs/zh/sql-reference/table-functions/postgresql.md
Normal file
@ -0,0 +1,120 @@
|
||||
---
|
||||
toc_priority: 42
|
||||
toc_title: postgresql
|
||||
---
|
||||
|
||||
# postgresql {#postgresql}
|
||||
|
||||
允许对存储在远程 PostgreSQL 服务器上的数据进行 `SELECT` 和 `INSERT` 查询.
|
||||
|
||||
**语法**
|
||||
|
||||
``` sql
|
||||
postgresql('host:port', 'database', 'table', 'user', 'password'[, `schema`])
|
||||
```
|
||||
|
||||
**参数**
|
||||
|
||||
- `host:port` — PostgreSQL 服务器地址.
|
||||
- `database` — 远程数据库名称.
|
||||
- `table` — 远程表名称.
|
||||
- `user` — PostgreSQL 用户.
|
||||
- `password` — 用户密码.
|
||||
- `schema` — 非默认的表结构. 可选.
|
||||
|
||||
**返回值**
|
||||
|
||||
一个表对象，其列数与原 PostgreSQL 表的列数相同。
|
||||
|
||||
!!! info "Note"
|
||||
在`INSERT`查询中，为了区分表函数`postgresql(..)`和表名以及表的列名列表，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。请看下面的例子。
|
||||
|
||||
## 实施细节 {#implementation-details}
|
||||
|
||||
`SELECT`查询在 PostgreSQL 上以 `COPY (SELECT ...) TO STDOUT` 的方式在只读的 PostgreSQL 事务中运行，每次在`SELECT`查询后提交。
|
||||
|
||||
简单的`WHERE`子句，如`=`、`！=`、`>`、`>=`、`<`、`<=`和`IN`，在PostgreSQL服务器上执行。
|
||||
|
||||
所有的连接、聚合、排序，`IN [ 数组 ]`条件和`LIMIT`采样约束只有在对PostgreSQL的查询结束后才会在ClickHouse中执行。
|
||||
|
||||
PostgreSQL 上的`INSERT`查询以`COPY "table_name" (field1, field2, ... fieldN) FROM STDIN`的方式在 PostgreSQL 事务中运行，每次`INSERT`语句后自动提交。
|
||||
|
||||
PostgreSQL 数组类型将转换为 ClickHouse 数组。
|
||||
|
||||
!!! info "Note"
|
||||
要小心，在 PostgreSQL 中，像 Integer[] 这样的数组数据类型列可以在不同的行中包含不同维度的数组，但在 ClickHouse 中，只允许在所有的行中有相同维度的多维数组。
|
||||
|
||||
支持设置 PostgreSQL 字典源中 Replicas 的优先级。地图中的数字越大，优先级就越低。`0`代表最高的优先级。
|
||||
|
||||
**示例**
|
||||
|
||||
PostgreSQL 中的表:
|
||||
|
||||
``` text
|
||||
postgres=# CREATE TABLE "public"."test" (
|
||||
"int_id" SERIAL,
|
||||
"int_nullable" INT NULL DEFAULT NULL,
|
||||
"float" FLOAT NOT NULL,
|
||||
"str" VARCHAR(100) NOT NULL DEFAULT '',
|
||||
"float_nullable" FLOAT NULL DEFAULT NULL,
|
||||
PRIMARY KEY (int_id));
|
||||
|
||||
CREATE TABLE
|
||||
|
||||
postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2);
|
||||
INSERT 0 1
|
||||
|
||||
postgresql> SELECT * FROM test;
|
||||
int_id | int_nullable | float | str | float_nullable
|
||||
--------+--------------+-------+------+----------------
|
||||
1 | | 2 | test |
|
||||
(1 row)
|
||||
```
|
||||
|
||||
从 ClickHouse 检索数据:
|
||||
|
||||
```sql
|
||||
SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password') WHERE str IN ('test');
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │
|
||||
└────────┴──────────────┴───────┴──────┴────────────────┘
|
||||
```
|
||||
|
||||
插入数据:
|
||||
|
||||
```sql
|
||||
INSERT INTO TABLE FUNCTION postgresql('localhost:5432', 'test', 'test', 'postgrsql_user', 'password') (int_id, float) VALUES (2, 3);
|
||||
SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password');
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │
|
||||
│ 2 │ ᴺᵁᴸᴸ │ 3 │ │ ᴺᵁᴸᴸ │
|
||||
└────────┴──────────────┴───────┴──────┴────────────────┘
|
||||
```
|
||||
|
||||
使用非默认的表结构:
|
||||
|
||||
```text
|
||||
postgres=# CREATE SCHEMA "nice.schema";
|
||||
|
||||
postgres=# CREATE TABLE "nice.schema"."nice.table" (a integer);
|
||||
|
||||
postgres=# INSERT INTO "nice.schema"."nice.table" SELECT i FROM generate_series(0, 99) as t(i)
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE pg_table_schema_with_dots (a UInt32)
|
||||
ENGINE PostgreSQL('localhost:5432', 'clickhouse', 'nice.table', 'postgrsql_user', 'password', 'nice.schema');
|
||||
```
|
||||
|
||||
**另请参阅**
|
||||
|
||||
- [PostgreSQL 表引擎](../../engines/table-engines/integrations/postgresql.md)
|
||||
- [使用 PostgreSQL 作为外部字典的来源](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql)
|
||||
|
||||
[原始文章](https://clickhouse.tech/docs/en/sql-reference/table-functions/postgresql/) <!--hide-->
|
||||
@ -50,11 +50,15 @@ option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories"
|
||||
|
||||
|
||||
option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
option (ENABLE_CLICKHOUSE_KEEPER_CONVERTER "Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot" ${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
if (NOT USE_NURAFT)
|
||||
# RECONFIGURE_MESSAGE_LEVEL should not be used here,
|
||||
# since USE_NURAFT is set to OFF for FreeBSD and Darwin.
|
||||
message (STATUS "clickhouse-keeper will not be built (lack of NuRaft)")
|
||||
message (STATUS "clickhouse-keeper and clickhouse-keeper-converter will not be built (lack of NuRaft)")
|
||||
set(ENABLE_CLICKHOUSE_KEEPER OFF)
|
||||
set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
|
||||
endif()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
@ -150,6 +154,12 @@ else()
|
||||
message(STATUS "ClickHouse keeper mode: OFF")
|
||||
endif()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
message(STATUS "ClickHouse keeper-converter mode: ON")
|
||||
else()
|
||||
message(STATUS "ClickHouse keeper-converter mode: OFF")
|
||||
endif()
|
||||
|
||||
if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
|
||||
set(CLICKHOUSE_ONE_SHARED ON)
|
||||
endif()
|
||||
@ -222,6 +232,10 @@ if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
add_subdirectory (keeper)
|
||||
endif()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
add_subdirectory (keeper-converter)
|
||||
endif()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
add_subdirectory (odbc-bridge)
|
||||
endif ()
|
||||
@ -231,9 +245,51 @@ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
|
||||
endif ()
|
||||
|
||||
if (CLICKHOUSE_ONE_SHARED)
|
||||
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES} ${CLICKHOUSE_KEEPER_SOURCES})
|
||||
target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK} ${CLICKHOUSE_KEEPER_LINK})
|
||||
target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} ${CLICKHOUSE_KEEPER_INCLUDE})
|
||||
add_library(clickhouse-lib SHARED
|
||||
${CLICKHOUSE_SERVER_SOURCES}
|
||||
${CLICKHOUSE_CLIENT_SOURCES}
|
||||
${CLICKHOUSE_LOCAL_SOURCES}
|
||||
${CLICKHOUSE_BENCHMARK_SOURCES}
|
||||
${CLICKHOUSE_COPIER_SOURCES}
|
||||
${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES}
|
||||
${CLICKHOUSE_COMPRESSOR_SOURCES}
|
||||
${CLICKHOUSE_FORMAT_SOURCES}
|
||||
${CLICKHOUSE_OBFUSCATOR_SOURCES}
|
||||
${CLICKHOUSE_GIT_IMPORT_SOURCES}
|
||||
${CLICKHOUSE_ODBC_BRIDGE_SOURCES}
|
||||
${CLICKHOUSE_KEEPER_SOURCES}
|
||||
${CLICKHOUSE_KEEPER_CONVERTER_SOURCES})
|
||||
|
||||
target_link_libraries(clickhouse-lib
|
||||
${CLICKHOUSE_SERVER_LINK}
|
||||
${CLICKHOUSE_CLIENT_LINK}
|
||||
${CLICKHOUSE_LOCAL_LINK}
|
||||
${CLICKHOUSE_BENCHMARK_LINK}
|
||||
${CLICKHOUSE_COPIER_LINK}
|
||||
${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK}
|
||||
${CLICKHOUSE_COMPRESSOR_LINK}
|
||||
${CLICKHOUSE_FORMAT_LINK}
|
||||
${CLICKHOUSE_OBFUSCATOR_LINK}
|
||||
${CLICKHOUSE_GIT_IMPORT_LINK}
|
||||
${CLICKHOUSE_ODBC_BRIDGE_LINK}
|
||||
${CLICKHOUSE_KEEPER_LINK}
|
||||
${CLICKHOUSE_KEEPER_CONVERTER_LINK})
|
||||
|
||||
target_include_directories(clickhouse-lib
|
||||
${CLICKHOUSE_SERVER_INCLUDE}
|
||||
${CLICKHOUSE_CLIENT_INCLUDE}
|
||||
${CLICKHOUSE_LOCAL_INCLUDE}
|
||||
${CLICKHOUSE_BENCHMARK_INCLUDE}
|
||||
${CLICKHOUSE_COPIER_INCLUDE}
|
||||
${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE}
|
||||
${CLICKHOUSE_COMPRESSOR_INCLUDE}
|
||||
${CLICKHOUSE_FORMAT_INCLUDE}
|
||||
${CLICKHOUSE_OBFUSCATOR_INCLUDE}
|
||||
${CLICKHOUSE_GIT_IMPORT_INCLUDE}
|
||||
${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}
|
||||
${CLICKHOUSE_KEEPER_INCLUDE}
|
||||
${CLICKHOUSE_KEEPER_CONVERTER_INCLUDE})
|
||||
|
||||
set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
|
||||
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
@ -264,6 +320,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
|
||||
endif ()
|
||||
|
||||
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
|
||||
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
|
||||
@ -314,6 +374,9 @@ else ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper-converter)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_INSTALL)
|
||||
clickhouse_target_link_split_lib(clickhouse install)
|
||||
endif ()
|
||||
@ -374,6 +437,11 @@ else ()
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
|
||||
endif ()
|
||||
|
||||
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
|
||||
|
||||
@ -1380,9 +1380,19 @@ private:
|
||||
have_error = true;
|
||||
}
|
||||
|
||||
const auto * exception = server_exception ? server_exception.get() : client_exception.get();
|
||||
// Sometimes you may get TOO_DEEP_RECURSION from the server,
|
||||
// and TOO_DEEP_RECURSION should not fail the fuzzer check.
|
||||
if (have_error && exception->code() == ErrorCodes::TOO_DEEP_RECURSION)
|
||||
{
|
||||
have_error = false;
|
||||
server_exception.reset();
|
||||
client_exception.reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (have_error)
|
||||
{
|
||||
const auto * exception = server_exception ? server_exception.get() : client_exception.get();
|
||||
fmt::print(stderr, "Error on processing query '{}': {}\n", ast_to_process->formatForErrorMessage(), exception->message());
|
||||
|
||||
// Try to reconnect after errors, for two reasons:
|
||||
|
||||
@ -17,3 +17,4 @@
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER_CONVERTER
|
||||
|
||||
9
programs/keeper-converter/CMakeLists.txt
Normal file
9
programs/keeper-converter/CMakeLists.txt
Normal file
@ -0,0 +1,9 @@
|
||||
set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
|
||||
|
||||
set (CLICKHOUSE_KEEPER_CONVERTER_LINK
|
||||
PRIVATE
|
||||
boost::program_options
|
||||
dbms
|
||||
)
|
||||
|
||||
clickhouse_program_add(keeper-converter)
|
||||
61
programs/keeper-converter/KeeperConverter.cpp
Normal file
61
programs/keeper-converter/KeeperConverter.cpp
Normal file
@ -0,0 +1,61 @@
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
#include <Coordination/KeeperSnapshotManager.h>
|
||||
#include <Coordination/ZooKeeperDataReader.h>
|
||||
#include <Common/TerminalSize.h>
|
||||
#include <Poco/ConsoleChannel.h>
|
||||
#include <Poco/AutoPtr.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
|
||||
int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
|
||||
{
|
||||
using namespace DB;
|
||||
namespace po = boost::program_options;
|
||||
|
||||
po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
|
||||
desc.add_options()
|
||||
("help,h", "produce help message")
|
||||
("zookeeper-logs-dir", po::value<std::string>(), "Path to directory with ZooKeeper logs")
|
||||
("zookeeper-snapshots-dir", po::value<std::string>(), "Path to directory with ZooKeeper snapshots")
|
||||
("output-dir", po::value<std::string>(), "Directory to place output clickhouse-keeper snapshot")
|
||||
;
|
||||
po::variables_map options;
|
||||
po::store(po::command_line_parser(argc, argv).options(desc).run(), options);
|
||||
Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
|
||||
|
||||
Poco::Logger * logger = &Poco::Logger::get("KeeperConverter");
|
||||
logger->setChannel(console_channel);
|
||||
|
||||
if (options.count("help"))
|
||||
{
|
||||
std::cout << "Usage: " << argv[0] << " --zookeeper-logs-dir /var/lib/zookeeper/data/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/data/version-2 --output-dir /var/lib/clickhouse/coordination/snapshots" << std::endl;
|
||||
std::cout << desc << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DB::KeeperStorage storage(500, "");
|
||||
|
||||
DB::deserializeKeeperStorageFromSnapshotsDir(storage, options["zookeeper-snapshots-dir"].as<std::string>(), logger);
|
||||
DB::deserializeLogsAndApplyToStorage(storage, options["zookeeper-logs-dir"].as<std::string>(), logger);
|
||||
DB::SnapshotMetadataPtr snapshot_meta = std::make_shared<DB::SnapshotMetadata>(storage.getZXID(), 1, std::make_shared<nuraft::cluster_config>());
|
||||
DB::KeeperStorageSnapshot snapshot(&storage, snapshot_meta);
|
||||
|
||||
DB::KeeperSnapshotManager manager(options["output-dir"].as<std::string>(), 1);
|
||||
auto snp = manager.serializeSnapshotToBuffer(snapshot);
|
||||
auto path = manager.serializeSnapshotBufferToDisk(*snp, storage.getZXID());
|
||||
std::cout << "Snapshot serialized to path:" << path << std::endl;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << getCurrentExceptionMessage(true) << '\n';
|
||||
return getCurrentExceptionCode();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -0,0 +1,2 @@
|
||||
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
|
||||
int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); }
|
||||
@ -59,6 +59,9 @@ int mainEntryClickHouseGitImport(int argc, char ** argv);
|
||||
#if ENABLE_CLICKHOUSE_KEEPER
|
||||
int mainEntryClickHouseKeeper(int argc, char ** argv);
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_KEEPER
|
||||
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_INSTALL
|
||||
int mainEntryClickHouseInstall(int argc, char ** argv);
|
||||
int mainEntryClickHouseStart(int argc, char ** argv);
|
||||
@ -119,6 +122,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
|
||||
#if ENABLE_CLICKHOUSE_KEEPER
|
||||
{"keeper", mainEntryClickHouseKeeper},
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_KEEPER_CONVERTER
|
||||
{"keeper-converter", mainEntryClickHouseKeeperConverter},
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_INSTALL
|
||||
{"install", mainEntryClickHouseInstall},
|
||||
{"start", mainEntryClickHouseStart},
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user