thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into retry-more-error-codes-from-s3

Browse Source
This commit is contained in:
Alexey Milovidov 2024-10-29 05:00:14 +01:00
commit 617775a704
12 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/sql-reference/data-types/newjson.md
View File

@ -5,7 +5,7 @@ sidebar_label: JSON
keywords: [json, data type]
---
# JSON
# JSON Data Type
Stores JavaScript Object Notation (JSON) documents in a single column.

2
programs/server/Server.cpp
View File

@ -168,6 +168,7 @@ namespace ServerSetting
{
extern const ServerSettingsUInt32 asynchronous_heavy_metrics_update_period_s;
extern const ServerSettingsUInt32 asynchronous_metrics_update_period_s;
extern const ServerSettingsBool asynchronous_metrics_enable_heavy_metrics;
extern const ServerSettingsBool async_insert_queue_flush_on_shutdown;
extern const ServerSettingsUInt64 async_insert_threads;
extern const ServerSettingsBool async_load_databases;
@ -1061,6 +1062,7 @@ try
ServerAsynchronousMetrics async_metrics(
global_context,
server_settings[ServerSetting::asynchronous_metrics_update_period_s],
server_settings[ServerSetting::asynchronous_metrics_enable_heavy_metrics],
server_settings[ServerSetting::asynchronous_heavy_metrics_update_period_s],
[&]() -> std::vector<ProtocolServerMetrics>
{

1
src/Core/ServerSettings.cpp
View File

@ -58,6 +58,7 @@ namespace DB
DECLARE(Double, cannot_allocate_thread_fault_injection_probability, 0, "For testing purposes.", 0) \
DECLARE(Int32, max_connections, 1024, "Max server connections.", 0) \
DECLARE(UInt32, asynchronous_metrics_update_period_s, 1, "Period in seconds for updating asynchronous metrics.", 0) \
DECLARE(Bool, asynchronous_metrics_enable_heavy_metrics, false, "Enable the calculation of heavy asynchronous metrics.", 0) \
DECLARE(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \
DECLARE(String, default_database, "default", "Default database name.", 0) \
DECLARE(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \

9
src/Interpreters/ServerAsynchronousMetrics.cpp
View File

@ -54,12 +54,14 @@ void calculateMaxAndSum(Max & max, Sum & sum, T x)
ServerAsynchronousMetrics::ServerAsynchronousMetrics(
ContextPtr global_context_,
unsigned update_period_seconds,
bool update_heavy_metrics_,
unsigned heavy_metrics_update_period_seconds,
const ProtocolServerMetricsFunc & protocol_server_metrics_func_,
bool update_jemalloc_epoch_,
bool update_rss_)
: WithContext(global_context_)
, AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, update_jemalloc_epoch_, update_rss_)
, update_heavy_metrics(update_heavy_metrics_)
, heavy_metric_update_period(heavy_metrics_update_period_seconds)
{
/// sanity check
@ -412,7 +414,8 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr
}
#endif
updateHeavyMetricsIfNeeded(current_time, update_time, force_update, first_run, new_values);
if (update_heavy_metrics)
updateHeavyMetricsIfNeeded(current_time, update_time, force_update, first_run, new_values);
}
void ServerAsynchronousMetrics::logImpl(AsynchronousMetricValues & new_values)
@ -459,10 +462,10 @@ void ServerAsynchronousMetrics::updateDetachedPartsStats()
void ServerAsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_time, TimePoint update_time, bool force_update, bool first_run, AsynchronousMetricValues & new_values)
{
const auto time_since_previous_update = current_time - heavy_metric_previous_update_time;
const bool update_heavy_metrics = (time_since_previous_update >= heavy_metric_update_period) || force_update || first_run;
const bool need_update_heavy_metrics = (time_since_previous_update >= heavy_metric_update_period) || force_update || first_run;
Stopwatch watch;
if (update_heavy_metrics)
if (need_update_heavy_metrics)
{
heavy_metric_previous_update_time = update_time;
if (first_run)

2
src/Interpreters/ServerAsynchronousMetrics.h
View File

@ -13,6 +13,7 @@ public:
ServerAsynchronousMetrics(
ContextPtr global_context_,
unsigned update_period_seconds,
bool update_heavy_metrics_,
unsigned heavy_metrics_update_period_seconds,
const ProtocolServerMetricsFunc & protocol_server_metrics_func_,
bool update_jemalloc_epoch_,
@ -24,6 +25,7 @@ private:
void updateImpl(TimePoint update_time, TimePoint current_time, bool force_update, bool first_run, AsynchronousMetricValues & new_values) override;
void logImpl(AsynchronousMetricValues & new_values) override;
bool update_heavy_metrics;
const Duration heavy_metric_update_period;
TimePoint heavy_metric_previous_update_time;
double heavy_update_interval = 0.;

3
src/Interpreters/SystemLog.cpp
View File

@ -298,9 +298,6 @@ SystemLogs::SystemLogs(ContextPtr global_context, const Poco::Util::AbstractConf
#undef CREATE_PUBLIC_MEMBERS
/// NOLINTEND(bugprone-macro-parentheses)
if (session_log)
global_context->addWarningMessage("Table system.session_log is enabled. It's unreliable and may contain garbage. Do not use it for any kind of security monitoring.");
bool should_prepare = global_context->getServerSettings()[ServerSetting::prepare_system_log_tables_on_startup];
try
{

2
src/Storages/System/StorageSystemLicenses.sh
View File

@ -1,5 +1,7 @@
#!/usr/bin/env bash
set -e -o pipefail
ROOT_PATH="$(git rev-parse --show-toplevel)"
IFS=$'\t'

2
src/Storages/System/attachSystemTables.cpp
View File

@ -126,8 +126,6 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b
attachNoDescription<StorageSystemOne>(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`.");
attachNoDescription<StorageSystemNumbers>(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number");
attachNoDescription<StorageSystemNumbers>(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number");
attachNoDescription<StorageSystemNumbers>(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series");
attachNoDescription<StorageSystemNumbers>(context, system_database, "generateSeries", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series");
attachNoDescription<StorageSystemZeros>(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false);
attachNoDescription<StorageSystemZeros>(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true);
attach<StorageSystemDatabases>(context, system_database, "databases", "Lists all databases of the current server.");

1
tests/integration/test_detached_parts_metrics/configs/asynchronous_metrics_update_period_s.xml
View File

@ -1,4 +1,5 @@
<clickhouse>
<asynchronous_metrics_update_period_s>1</asynchronous_metrics_update_period_s>
<asynchronous_metrics_enable_heavy_metrics>1</asynchronous_metrics_enable_heavy_metrics>
<asynchronous_heavy_metrics_update_period_s>1</asynchronous_heavy_metrics_update_period_s>
</clickhouse>

1
tests/queries/0_stateless/02932_refreshable_materialized_views_1.reference
View File

@ -2,7 +2,6 @@
CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 SECOND\n(\n `x` UInt64\n)\nENGINE = Memory\nAS SELECT number AS x\nFROM numbers(2)\nUNION ALL\nSELECT rand64() AS x
<2: refreshed> 3 1 1
<3: time difference at least> 1000
<4: next refresh in> 2 Scheduled
<4.1: fake clock> Scheduled 2050-01-01 00:00:01 2050-01-01 00:00:02 1 3 3 3 0
<4.5: altered> Scheduled 2050-01-01 00:00:01 2052-01-01 00:00:00
CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 YEAR\n(\n `x` UInt64\n)\nENGINE = Memory\nAS SELECT x * 2 AS x\nFROM default.src

8
tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh
View File

@ -50,14 +50,6 @@ done
# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above.
$CLICKHOUSE_CLIENT -q "
select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000);"
while :
do
# Wait for status to change to Scheduled. If status = Scheduling, next_refresh_time is stale.
res="`$CLICKHOUSE_CLIENT -q "select '<4: next refresh in>', next_refresh_time-last_success_time, status from refreshes -- $LINENO"`"
echo "$res" | grep -q 'Scheduled' && break
sleep 0.5
done
echo "$res"
# Create a source table from which views will read.
$CLICKHOUSE_CLIENT -q "

36
utils/list-licenses/list-licenses.sh
View File

@ -1,6 +1,7 @@
#!/usr/bin/env bash
if [[ "$OSTYPE" == "darwin"* ]]; then
if [[ "$OSTYPE" == "darwin"* ]]
then
# use GNU versions, their presence is ensured in cmake/tools.cmake
GREP_CMD=ggrep
FIND_CMD=gfind
@ -12,31 +13,35 @@ fi
ROOT_PATH="$(git rev-parse --show-toplevel)"
LIBS_PATH="${ROOT_PATH}/contrib"
mapfile -t libs < <(echo "${ROOT_PATH}/base/poco"; find "${LIBS_PATH}" -type d -maxdepth 1 ! -name '*-cmake' | LC_ALL=C sort)
for LIB in "${libs[@]}"; do
mapfile -t libs < <(echo "${ROOT_PATH}/base/poco"; find "${LIBS_PATH}" -maxdepth 1 -type d -not -name '*-cmake' -not -name 'rust_vendor' | LC_ALL=C sort)
for LIB in "${libs[@]}"
do
LIB_NAME=$(basename "$LIB")
LIB_LICENSE=$(
LC_ALL=C ${FIND_CMD} "$LIB" -type f -and '(' -iname 'LICENSE*' -or -iname 'COPYING*' -or -iname 'COPYRIGHT*' ')' -and -not '(' -iname '*.html' -or -iname '*.htm' -or -iname '*.rtf' -or -name '*.cpp' -or -name '*.h' -or -iname '*.json' ')' -printf "%d\t%p\n" |
LC_ALL=C ${FIND_CMD} "$LIB" -type f -and '(' -iname 'LICENSE*' -or -iname 'COPYING*' -or -iname 'COPYRIGHT*' -or -iname 'NOTICE' ')' -and -not '(' -iname '*.html' -or -iname '*.htm' -or -iname '*.rtf' -or -name '*.cpp' -or -name '*.h' -or -iname '*.json' ')' -printf "%d\t%p\n" |
LC_ALL=C sort | LC_ALL=C awk '
BEGIN { IGNORECASE=1; min_depth = 0 }
/LICENSE/ { if (!min_depth || $1 <= min_depth) { min_depth = $1; license = $2 } }
/COPY/ { if (!min_depth || $1 <= min_depth) { min_depth = $1; copying = $2 } }
END { if (license) { print license } else { print copying } }')
if [ -n "$LIB_LICENSE" ]; then
/NOTICE/ { if (!min_depth || $1 <= min_depth) { min_depth = $1; notice = $2 } }
END { if (license) { print license } else if (copying) { print copying } else { print notice } }')
if [ -n "$LIB_LICENSE" ]
then
LICENSE_TYPE=$(
(${GREP_CMD} -q -F 'Apache' "$LIB_LICENSE" &&
echo "Apache") ||
(${GREP_CMD} -q -F 'Boost' "$LIB_LICENSE" &&
echo "Boost") ||
(${GREP_CMD} -q -i -P 'public\s*domain' "$LIB_LICENSE" &&
(${GREP_CMD} -q -i -P 'public\s*domain|CC0 1\.0 Universal' "$LIB_LICENSE" &&
echo "Public Domain") ||
(${GREP_CMD} -q -F 'BSD' "$LIB_LICENSE" &&
echo "BSD") ||
(${GREP_CMD} -q -F 'Lesser General Public License' "$LIB_LICENSE" &&
echo "LGPL") ||
(${GREP_CMD} -q -F 'General Public License' "$LIB_LICENSE" &&
echo "GPL") ||
(${GREP_CMD} -q -i -F 'The origin of this software must not be misrepresented' "$LIB_LICENSE" &&
${GREP_CMD} -q -i -F 'Altered source versions must be plainly marked as such' "$LIB_LICENSE" &&
${GREP_CMD} -q -i -F 'This notice may not be removed or altered' "$LIB_LICENSE" &&
@ -73,8 +78,23 @@ for LIB in "${libs[@]}"; do
echo "HPND") ||
echo "Unknown")
if [ "$LICENSE_TYPE" == "GPL" ]
then
echo "Fatal error: General Public License found in ${LIB_NAME}." >&2
exit 1
fi
if [ "$LICENSE_TYPE" == "Unknown" ]
then
echo "Fatal error: sources with unknown license found in ${LIB_NAME}." >&2
exit 1
fi
RELATIVE_PATH=$(echo "$LIB_LICENSE" | sed -r -e 's!^.+/(contrib|base)/!/\1/!')
echo -e "$LIB_NAME\t$LICENSE_TYPE\t$RELATIVE_PATH"
fi
done
# Special care for Rust
find "${LIBS_PATH}/rust_vendor/" -name 'Cargo.toml' | xargs grep 'license = ' | (grep -v -P 'MIT|Apache|MPL' && echo "Fatal error: unrecognized licenses in the Rust code" >&2 && exit 1 || true)