mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-23 08:02:02 +00:00
Merge branch 'master' into zk-retry-fix
This commit is contained in:
Igor Nikonov
GitHub
commit
6abd1a6462
2
contrib/croaring
vendored
2
contrib/croaring
vendored
@ -1 +1 @@
|
||||
Subproject commit e4a7ad5542746103e71ca8b5e56225baf0014c87
|
||||
Subproject commit 9b7cc0ff1c41e9457efb6228cfd2c538d0155303
|
||||
2
contrib/curl
vendored
2
contrib/curl
vendored
@ -1 +1 @@
|
||||
Subproject commit eb3b049df526bf125eda23218e680ce7fa9ec46c
|
||||
Subproject commit d755a5f7c009dd63a61b2c745180d8ba937cbfeb
|
||||
@ -64,6 +64,7 @@ set (SRCS
|
||||
"${LIBRARY_DIR}/lib/hostsyn.c"
|
||||
"${LIBRARY_DIR}/lib/hsts.c"
|
||||
"${LIBRARY_DIR}/lib/http.c"
|
||||
"${LIBRARY_DIR}/lib/http1.c"
|
||||
"${LIBRARY_DIR}/lib/http2.c"
|
||||
"${LIBRARY_DIR}/lib/http_aws_sigv4.c"
|
||||
"${LIBRARY_DIR}/lib/http_chunks.c"
|
||||
|
||||
@ -6,20 +6,14 @@ target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
|
||||
target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/googletest/include")
|
||||
target_include_directories(_gtest PRIVATE "${SRC_DIR}/googletest")
|
||||
|
||||
add_library(_gtest_main "${SRC_DIR}/googletest/src/gtest_main.cc")
|
||||
set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0")
|
||||
target_link_libraries(_gtest_main PUBLIC _gtest)
|
||||
|
||||
add_library(_gtest_all INTERFACE)
|
||||
target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main)
|
||||
add_library(ch_contrib::gtest_all ALIAS _gtest_all)
|
||||
|
||||
add_library(ch_contrib::gtest ALIAS _gtest)
|
||||
|
||||
add_library(_gmock "${SRC_DIR}/googlemock/src/gmock-all.cc")
|
||||
set_target_properties(_gmock PROPERTIES VERSION "1.0.0")
|
||||
target_compile_definitions (_gmock PUBLIC GTEST_HAS_POSIX_RE=0)
|
||||
target_include_directories(_gmock SYSTEM PUBLIC "${SRC_DIR}/googlemock/include" "${SRC_DIR}/googletest/include")
|
||||
target_include_directories(_gmock PRIVATE "${SRC_DIR}/googlemock")
|
||||
target_link_libraries(_gmock PUBLIC _gtest)
|
||||
|
||||
add_library(_gmock_main "${SRC_DIR}/googlemock/src/gmock_main.cc")
|
||||
set_target_properties(_gmock_main PROPERTIES VERSION "1.0.0")
|
||||
|
||||
@ -104,66 +104,76 @@ if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CL
|
||||
EOT
|
||||
fi
|
||||
|
||||
if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then
|
||||
# port is needed to check if clickhouse-server is ready for connections
|
||||
HTTP_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=http_port --try)"
|
||||
HTTPS_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=https_port --try)"
|
||||
# checking $DATA_DIR for initialization
|
||||
if [ -d "${DATA_DIR%/}/data" ]; then
|
||||
DATABASE_ALREADY_EXISTS='true'
|
||||
fi
|
||||
|
||||
if [ -n "$HTTP_PORT" ]; then
|
||||
URL="http://127.0.0.1:$HTTP_PORT/ping"
|
||||
else
|
||||
URL="https://127.0.0.1:$HTTPS_PORT/ping"
|
||||
fi
|
||||
# only run initialization on an empty data directory
|
||||
if [ -z "${DATABASE_ALREADY_EXISTS}" ]; then
|
||||
if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then
|
||||
# port is needed to check if clickhouse-server is ready for connections
|
||||
HTTP_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=http_port --try)"
|
||||
HTTPS_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=https_port --try)"
|
||||
|
||||
# Listen only on localhost until the initialization is done
|
||||
/usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
|
||||
pid="$!"
|
||||
if [ -n "$HTTP_PORT" ]; then
|
||||
URL="http://127.0.0.1:$HTTP_PORT/ping"
|
||||
else
|
||||
URL="https://127.0.0.1:$HTTPS_PORT/ping"
|
||||
fi
|
||||
|
||||
# check if clickhouse is ready to accept connections
|
||||
# will try to send ping clickhouse via http_port (max 1000 retries by default, with 1 sec timeout and 1 sec delay between retries)
|
||||
tries=${CLICKHOUSE_INIT_TIMEOUT:-1000}
|
||||
while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do
|
||||
if [ "$tries" -le "0" ]; then
|
||||
echo >&2 'ClickHouse init process failed.'
|
||||
# Listen only on localhost until the initialization is done
|
||||
/usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
|
||||
pid="$!"
|
||||
|
||||
# check if clickhouse is ready to accept connections
|
||||
# will try to send ping clickhouse via http_port (max 1000 retries by default, with 1 sec timeout and 1 sec delay between retries)
|
||||
tries=${CLICKHOUSE_INIT_TIMEOUT:-1000}
|
||||
while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do
|
||||
if [ "$tries" -le "0" ]; then
|
||||
echo >&2 'ClickHouse init process failed.'
|
||||
exit 1
|
||||
fi
|
||||
tries=$(( tries-1 ))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
clickhouseclient=( clickhouse-client --multiquery --host "127.0.0.1" -u "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" )
|
||||
|
||||
echo
|
||||
|
||||
# create default database, if defined
|
||||
if [ -n "$CLICKHOUSE_DB" ]; then
|
||||
echo "$0: create database '$CLICKHOUSE_DB'"
|
||||
"${clickhouseclient[@]}" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB";
|
||||
fi
|
||||
|
||||
for f in /docker-entrypoint-initdb.d/*; do
|
||||
case "$f" in
|
||||
*.sh)
|
||||
if [ -x "$f" ]; then
|
||||
echo "$0: running $f"
|
||||
"$f"
|
||||
else
|
||||
echo "$0: sourcing $f"
|
||||
# shellcheck source=/dev/null
|
||||
. "$f"
|
||||
fi
|
||||
;;
|
||||
*.sql) echo "$0: running $f"; "${clickhouseclient[@]}" < "$f" ; echo ;;
|
||||
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${clickhouseclient[@]}"; echo ;;
|
||||
*) echo "$0: ignoring $f" ;;
|
||||
esac
|
||||
echo
|
||||
done
|
||||
|
||||
if ! kill -s TERM "$pid" || ! wait "$pid"; then
|
||||
echo >&2 'Finishing of ClickHouse init process failed.'
|
||||
exit 1
|
||||
fi
|
||||
tries=$(( tries-1 ))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
clickhouseclient=( clickhouse-client --multiquery --host "127.0.0.1" -u "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" )
|
||||
|
||||
echo
|
||||
|
||||
# create default database, if defined
|
||||
if [ -n "$CLICKHOUSE_DB" ]; then
|
||||
echo "$0: create database '$CLICKHOUSE_DB'"
|
||||
"${clickhouseclient[@]}" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB";
|
||||
fi
|
||||
|
||||
for f in /docker-entrypoint-initdb.d/*; do
|
||||
case "$f" in
|
||||
*.sh)
|
||||
if [ -x "$f" ]; then
|
||||
echo "$0: running $f"
|
||||
"$f"
|
||||
else
|
||||
echo "$0: sourcing $f"
|
||||
# shellcheck source=/dev/null
|
||||
. "$f"
|
||||
fi
|
||||
;;
|
||||
*.sql) echo "$0: running $f"; "${clickhouseclient[@]}" < "$f" ; echo ;;
|
||||
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${clickhouseclient[@]}"; echo ;;
|
||||
*) echo "$0: ignoring $f" ;;
|
||||
esac
|
||||
echo
|
||||
done
|
||||
|
||||
if ! kill -s TERM "$pid" || ! wait "$pid"; then
|
||||
echo >&2 'Finishing of ClickHouse init process failed.'
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "ClickHouse Database directory appears to contain a database; Skipping initialization"
|
||||
fi
|
||||
|
||||
# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments
|
||||
|
||||
@ -15,8 +15,8 @@ CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
|
||||
# Pre-configured destination cluster, where to export the data
|
||||
CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
|
||||
|
||||
EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, "}
|
||||
EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
|
||||
EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name String, instance_type String, instance_id String, "}
|
||||
EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
|
||||
EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name, "}
|
||||
|
||||
function __set_connection_args
|
||||
@ -125,9 +125,9 @@ function setup_logs_replication
|
||||
echo 'Create %_log tables'
|
||||
clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
|
||||
do
|
||||
# Calculate hash of its structure. Note: 1 is the version of extra columns - increment it if extra columns are changed:
|
||||
# Calculate hash of its structure. Note: 4 is the version of extra columns - increment it if extra columns are changed:
|
||||
hash=$(clickhouse-client --query "
|
||||
SELECT sipHash64(1, groupArray((name, type)))
|
||||
SELECT sipHash64(4, groupArray((name, type)))
|
||||
FROM (SELECT name, type FROM system.columns
|
||||
WHERE database = 'system' AND table = '$table'
|
||||
ORDER BY position)
|
||||
|
||||
@ -10,6 +10,5 @@ RUN curl -L -o /mysql-connector-j-${ver}.jar https://repo1.maven.org/maven2/com/
|
||||
ENV CLASSPATH=$CLASSPATH:/mysql-connector-j-${ver}.jar
|
||||
|
||||
WORKDIR /jdbc
|
||||
COPY Test.java Test.java
|
||||
COPY PreparedStatementsTest.java PreparedStatementsTest.java
|
||||
RUN javac Test.java PreparedStatementsTest.java
|
||||
COPY MySQLJavaClientTest.java MySQLJavaClientTest.java
|
||||
RUN javac MySQLJavaClientTest.java
|
||||
|
||||
@ -2,7 +2,7 @@ import com.mysql.cj.MysqlType;
|
||||
|
||||
import java.sql.*;
|
||||
|
||||
public class PreparedStatementsTest {
|
||||
public class MySQLJavaClientTest {
|
||||
public static void main(String[] args) {
|
||||
int i = 0;
|
||||
String host = "127.0.0.1";
|
||||
@ -10,6 +10,7 @@ public class PreparedStatementsTest {
|
||||
String user = "default";
|
||||
String password = "";
|
||||
String database = "default";
|
||||
String binary = "false";
|
||||
while (i < args.length) {
|
||||
switch (args[i]) {
|
||||
case "--host":
|
||||
@ -27,16 +28,19 @@ public class PreparedStatementsTest {
|
||||
case "--database":
|
||||
database = args[++i];
|
||||
break;
|
||||
case "--binary":
|
||||
binary = args[++i];
|
||||
break;
|
||||
default:
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// useServerPrepStmts uses COM_STMT_PREPARE and COM_STMT_EXECUTE
|
||||
// instead of COM_QUERY which allows us to test the binary protocol
|
||||
String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=true",
|
||||
host, port, database);
|
||||
// useServerPrepStmts=true -> COM_STMT_PREPARE + COM_STMT_EXECUTE -> binary
|
||||
// useServerPrepStmts=false -> COM_QUERY -> text
|
||||
String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=%s",
|
||||
host, port, database, binary);
|
||||
|
||||
try {
|
||||
Class.forName("com.mysql.cj.jdbc.Driver");
|
||||
@ -49,6 +53,7 @@ public class PreparedStatementsTest {
|
||||
testDateTypes(conn);
|
||||
testUnusualDateTime64Scales(conn);
|
||||
testDateTimeTimezones(conn);
|
||||
testSuspiciousNullableLowCardinalityTypes(conn);
|
||||
conn.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
@ -58,7 +63,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testSimpleDataTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testSimpleDataTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_simple_data_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM simple_data_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -83,7 +88,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testStringTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testStringTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_string_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM string_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -97,7 +102,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testLowCardinalityAndNullableTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testLowCardinalityAndNullableTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_low_cardinality_and_nullable_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM low_cardinality_and_nullable_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -111,7 +116,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testDecimalTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testDecimalTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_decimal_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM decimal_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -127,7 +132,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testDateTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testDateTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_date_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM date_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -143,7 +148,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testUnusualDateTime64Scales(Connection conn) throws SQLException {
|
||||
System.out.println("### testUnusualDateTime64Scales");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_unusual_datetime64_scales").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM unusual_datetime64_scales").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -160,7 +165,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testDateTimeTimezones(Connection conn) throws SQLException {
|
||||
System.out.println("### testDateTimeTimezones");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_datetime_timezones").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM datetime_timezones").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -172,7 +177,7 @@ public class PreparedStatementsTest {
|
||||
|
||||
private static void testMiscTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testMiscTypes");
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM ps_misc_types").executeQuery();
|
||||
ResultSet rs = conn.prepareStatement("SELECT * FROM misc_types").executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
@ -184,6 +189,20 @@ public class PreparedStatementsTest {
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
private static void testSuspiciousNullableLowCardinalityTypes(Connection conn) throws SQLException {
|
||||
System.out.println("### testSuspiciousNullableLowCardinalityTypes");
|
||||
String query = "SELECT * FROM suspicious_nullable_low_cardinality_types";
|
||||
ResultSet rs = conn.prepareStatement(query).executeQuery();
|
||||
int rowNum = 1;
|
||||
while (rs.next()) {
|
||||
System.out.printf("Row #%d\n", rowNum++);
|
||||
System.out.printf("%s, value: %s\n", getMysqlType(rs, "f"), rs.getFloat("f"));
|
||||
System.out.printf("%s, value: %s\n", getMysqlType(rs, "d"), rs.getDate("d"));
|
||||
System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
|
||||
}
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
private static String getMysqlType(ResultSet rs, String columnLabel) throws SQLException {
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
return String.format("%s type is %s", columnLabel,
|
||||
@ -1,78 +0,0 @@
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
class JavaConnectorTest {
|
||||
private static final String CREATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS default.test1 (`age` Int32, `name` String, `int_nullable` Nullable(Int32)) Engine = Memory";
|
||||
private static final String INSERT_SQL = "INSERT INTO default.test1(`age`, `name`) VALUES(33, 'jdbc'),(44, 'ck')";
|
||||
private static final String SELECT_SQL = "SELECT * FROM default.test1";
|
||||
private static final String SELECT_NUMBER_SQL = "SELECT * FROM system.numbers LIMIT 13";
|
||||
private static final String DROP_TABLE_SQL = "DROP TABLE default.test1";
|
||||
|
||||
public static void main(String[] args) {
|
||||
int i = 0;
|
||||
String host = "127.0.0.1";
|
||||
String port = "9004";
|
||||
String user = "default";
|
||||
String password = "";
|
||||
String database = "default";
|
||||
while (i < args.length) {
|
||||
switch (args[i]) {
|
||||
case "--host":
|
||||
host = args[++i];
|
||||
break;
|
||||
case "--port":
|
||||
port = args[++i];
|
||||
break;
|
||||
case "--user":
|
||||
user = args[++i];
|
||||
break;
|
||||
case "--password":
|
||||
password = args[++i];
|
||||
break;
|
||||
case "--database":
|
||||
database = args[++i];
|
||||
break;
|
||||
default:
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false", host, port, database);
|
||||
|
||||
Connection conn = null;
|
||||
Statement stmt = null;
|
||||
try {
|
||||
Class.forName("com.mysql.cj.jdbc.Driver");
|
||||
conn = DriverManager.getConnection(jdbcUrl, user, password);
|
||||
stmt = conn.createStatement();
|
||||
stmt.executeUpdate(CREATE_TABLE_SQL);
|
||||
stmt.executeUpdate(INSERT_SQL);
|
||||
|
||||
ResultSet rs = stmt.executeQuery(SELECT_SQL);
|
||||
while (rs.next()) {
|
||||
System.out.print(rs.getString("age"));
|
||||
System.out.print(rs.getString("name"));
|
||||
System.out.print(rs.getString("int_nullable"));
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
stmt.executeUpdate(DROP_TABLE_SQL);
|
||||
|
||||
rs = stmt.executeQuery(SELECT_NUMBER_SQL);
|
||||
while (rs.next()) {
|
||||
System.out.print(rs.getString(1));
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
stmt.close();
|
||||
conn.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -101,8 +101,6 @@ The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is
|
||||
|
||||
No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
|
||||
|
||||
Always execute `OPTIMIZE ... FINAL CLEANUP` and `OPTIMIZE ... FINAL` to delete rows with `is_deleted=1` defined, especially when you wish to insert a new row with an older version. Otherwise, the new row with an older version will be replaced and not be persisted.
|
||||
|
||||
:::
|
||||
|
||||
Example:
|
||||
|
||||
@ -10,10 +10,6 @@ ClickHouse supports the MySQL wire protocol. This allow tools that are MySQL-com
|
||||
|
||||
## Enabling the MySQL Interface On ClickHouse Cloud
|
||||
|
||||
:::note
|
||||
The MySQL interface for ClickHouse Cloud is currently in private preview. Please contact support@clickhouse.com to enable this feature for your ClickHouse Cloud service.
|
||||
:::
|
||||
|
||||
1. After creating your ClickHouse Cloud Service, on the credentials screen, select the MySQL tab
|
||||
|
||||
|
||||
|
||||
@ -44,6 +44,8 @@ ClickHouse Inc does **not** maintain the libraries listed below and hasn’t don
|
||||
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
|
||||
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
|
||||
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
|
||||
- [clickhouse-ts](https://github.com/bytadaniel/clickhouse-ts)
|
||||
- [clickcache](https://github.com/bytadaniel/clickcache)
|
||||
### Perl
|
||||
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
|
||||
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
|
||||
|
||||
@ -61,17 +61,17 @@ FROM table
|
||||
SETTINGS use_query_cache = true, enable_writes_to_query_cache = false;
|
||||
```
|
||||
|
||||
For maximum control, it is generally recommended to provide settings "use_query_cache", "enable_writes_to_query_cache" and
|
||||
"enable_reads_from_query_cache" only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
|
||||
For maximum control, it is generally recommended to provide settings `use_query_cache`, `enable_writes_to_query_cache` and
|
||||
`enable_reads_from_query_cache` only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
|
||||
use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables
|
||||
may return cached results then.
|
||||
|
||||
The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table
|
||||
`system.query_cache`. The number of query cache hits and misses since database start are shown as events "QueryCacheHits" and
|
||||
"QueryCacheMisses" in system table [system.events](system-tables/events.md). Both counters are only updated for `SELECT` queries which run
|
||||
with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_cache_usage` in system table
|
||||
[system.query_log](system-tables/query_log.md) shows for each executed query whether the query result was written into or read from the
|
||||
query cache. Asynchronous metrics "QueryCacheEntries" and "QueryCacheBytes" in system table
|
||||
[system.query_cache](system-tables/query_cache.md). The number of query cache hits and misses since database start are shown as events
|
||||
"QueryCacheHits" and "QueryCacheMisses" in system table [system.events](system-tables/events.md). Both counters are only updated for
|
||||
`SELECT` queries which run with setting `use_query_cache = true`, other queries do not affect "QueryCacheMisses". Field `query_cache_usage`
|
||||
in system table [system.query_log](system-tables/query_log.md) shows for each executed query whether the query result was written into or
|
||||
read from the query cache. Asynchronous metrics "QueryCacheEntries" and "QueryCacheBytes" in system table
|
||||
[system.asynchronous_metrics](system-tables/asynchronous_metrics.md) show how many entries / bytes the query cache currently contains.
|
||||
|
||||
The query cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can be
|
||||
@ -86,9 +86,18 @@ If the query was aborted due to an exception or user cancellation, no entry is w
|
||||
The size of the query cache in bytes, the maximum number of cache entries and the maximum size of individual cache entries (in bytes and in
|
||||
records) can be configured using different [server configuration options](server-configuration-parameters/settings.md#server_configuration_parameters_query-cache).
|
||||
|
||||
```xml
|
||||
<query_cache>
|
||||
<max_size_in_bytes>1073741824</max_size_in_bytes>
|
||||
<max_entries>1024</max_entries>
|
||||
<max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
|
||||
<max_entry_size_in_rows>30000000</max_entry_size_in_rows>
|
||||
</query_cache>
|
||||
```
|
||||
|
||||
It is also possible to limit the cache usage of individual users using [settings profiles](settings/settings-profiles.md) and [settings
|
||||
constraints](settings/constraints-on-settings.md). More specifically, you can restrict the maximum amount of memory (in bytes) a user may
|
||||
allocate in the query cache and the the maximum number of stored query results. For that, first provide configurations
|
||||
allocate in the query cache and the maximum number of stored query results. For that, first provide configurations
|
||||
[query_cache_max_size_in_bytes](settings/settings.md#query-cache-max-size-in-bytes) and
|
||||
[query_cache_max_entries](settings/settings.md#query-cache-size-max-entries) in a user profile in `users.xml`, then make both settings
|
||||
readonly:
|
||||
@ -158,6 +167,7 @@ Also, results of queries with non-deterministic functions are not cached by defa
|
||||
- functions which depend on the environment: [`currentUser()`](../sql-reference/functions/other-functions.md#currentUser),
|
||||
[`queryID()`](../sql-reference/functions/other-functions.md#queryID),
|
||||
[`getMacro()`](../sql-reference/functions/other-functions.md#getMacro) etc.
|
||||
|
||||
To force caching of results of queries with non-deterministic functions regardless, use setting
|
||||
[query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
|
||||
|
||||
|
||||
@ -2403,7 +2403,8 @@ This section contains the following parameters:
|
||||
- zookeeper_load_balancing - Specifies the algorithm of ZooKeeper node selection.
|
||||
* random - randomly selects one of ZooKeeper nodes.
|
||||
* in_order - selects the first ZooKeeper node, if it's not available then the second, and so on.
|
||||
* nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname.
|
||||
* nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname, hostname is compared with name prefix.
|
||||
* hostname_levenshtein_distance - just like nearest_hostname, but it compares hostname in a levenshtein distance manner.
|
||||
* first_or_random - selects the first ZooKeeper node, if it's not available then randomly selects one of remaining ZooKeeper nodes.
|
||||
* round_robin - selects the first ZooKeeper node, if reconnection happens selects the next.
|
||||
|
||||
@ -2425,7 +2426,7 @@ This section contains the following parameters:
|
||||
<root>/path/to/zookeeper/node</root>
|
||||
<!-- Optional. Zookeeper digest ACL string. -->
|
||||
<identity>user:password</identity>
|
||||
<!--<zookeeper_load_balancing>random / in_order / nearest_hostname / first_or_random / round_robin</zookeeper_load_balancing>-->
|
||||
<!--<zookeeper_load_balancing>random / in_order / nearest_hostname / hostname_levenshtein_distance / first_or_random / round_robin</zookeeper_load_balancing>-->
|
||||
<zookeeper_load_balancing>random</zookeeper_load_balancing>
|
||||
</zookeeper>
|
||||
```
|
||||
|
||||
@ -1413,6 +1413,7 @@ ClickHouse supports the following algorithms of choosing replicas:
|
||||
|
||||
- [Random](#load_balancing-random) (by default)
|
||||
- [Nearest hostname](#load_balancing-nearest_hostname)
|
||||
- [Hostname levenshtein distance](#load_balancing-hostname_levenshtein_distance)
|
||||
- [In order](#load_balancing-in_order)
|
||||
- [First or random](#load_balancing-first_or_random)
|
||||
- [Round robin](#load_balancing-round_robin)
|
||||
@ -1444,6 +1445,25 @@ This method might seem primitive, but it does not require external data about ne
|
||||
Thus, if there are equivalent replicas, the closest one by name is preferred.
|
||||
We can also assume that when sending a query to the same server, in the absence of failures, a distributed query will also go to the same servers. So even if different data is placed on the replicas, the query will return mostly the same results.
|
||||
|
||||
### Hostname levenshtein distance {#load_balancing-hostname_levenshtein_distance}
|
||||
|
||||
``` sql
|
||||
load_balancing = hostname_levenshtein_distance
|
||||
```
|
||||
|
||||
Just like `nearest_hostname`, but it compares hostname in a [levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) manner. For example:
|
||||
|
||||
``` text
|
||||
example-clickhouse-0-0 ample-clickhouse-0-0
|
||||
1
|
||||
|
||||
example-clickhouse-0-0 example-clickhouse-1-10
|
||||
2
|
||||
|
||||
example-clickhouse-0-0 example-clickhouse-12-0
|
||||
3
|
||||
```
|
||||
|
||||
### In Order {#load_balancing-in_order}
|
||||
|
||||
``` sql
|
||||
@ -3281,7 +3301,7 @@ Default value: `0`.
|
||||
|
||||
## use_mysql_types_in_show_columns {#use_mysql_types_in_show_columns}
|
||||
|
||||
Show the names of MySQL data types corresponding to ClickHouse data types in [SHOW COLUMNS](../../sql-reference/statements/show.md#show_columns) and SELECTs on [system.columns](../system-tables/columns.md).
|
||||
Show the names of MySQL data types corresponding to ClickHouse data types in [SHOW COLUMNS](../../sql-reference/statements/show.md#show_columns).
|
||||
|
||||
Possible values:
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ The `system.columns` table contains the following columns (the column type is sh
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — Database name.
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Column name.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Column type. If setting `[use_mysql_types_in_show_columns](../../operations/settings/settings.md#use_mysql_types_in_show_columns) = 1` (default: 0), then the equivalent type name in MySQL is shown.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Column type.
|
||||
- `position` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Ordinal position of a column in a table starting with 1.
|
||||
- `default_kind` ([String](../../sql-reference/data-types/string.md)) — Expression type (`DEFAULT`, `MATERIALIZED`, `ALIAS`) for the default value, or an empty string if it is not defined.
|
||||
- `default_expression` ([String](../../sql-reference/data-types/string.md)) — Expression for the default value, or an empty string if it is not defined.
|
||||
|
||||
@ -11,21 +11,21 @@ The `system.part_log` table contains the following columns:
|
||||
|
||||
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Identifier of the `INSERT` query that created this data part.
|
||||
- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event that occurred with the data part. Can have one of the following values:
|
||||
- `NEW_PART` — Inserting of a new data part.
|
||||
- `MERGE_PARTS` — Merging of data parts.
|
||||
- `DOWNLOAD_PART` — Downloading a data part.
|
||||
- `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
|
||||
- `MUTATE_PART` — Mutating of a data part.
|
||||
- `MOVE_PART` — Moving the data part from the one disk to another one.
|
||||
- `NewPart` — Inserting of a new data part.
|
||||
- `MergeParts` — Merging of data parts.
|
||||
- `DownloadParts` — Downloading a data part.
|
||||
- `RemovePart` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
|
||||
- `MutatePart` — Mutating of a data part.
|
||||
- `MovePart` — Moving the data part from the one disk to another one.
|
||||
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values:
|
||||
- `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`.
|
||||
- `REGULAR_MERGE` — Some regular merge.
|
||||
- `TTL_DELETE_MERGE` — Cleaning up expired data.
|
||||
- `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
|
||||
- `NotAMerge` — The current event has the type other than `MERGE_PARTS`.
|
||||
- `RegularMerge` — Some regular merge.
|
||||
- `TTLDeleteMerge` — Cleaning up expired data.
|
||||
- `TTLRecompressMerge` — Recompressing data part with the.
|
||||
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
|
||||
- `UNDECIDED`
|
||||
- `HORIZONTAL`
|
||||
- `VERTICAL`
|
||||
- `Undecided`
|
||||
- `Horizontal`
|
||||
- `Vertical`
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
|
||||
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
|
||||
|
||||
36
docs/en/operations/system-tables/query_cache.md
Normal file
36
docs/en/operations/system-tables/query_cache.md
Normal file
@ -0,0 +1,36 @@
|
||||
---
|
||||
slug: /en/operations/system-tables/query_cache
|
||||
---
|
||||
# query_cache
|
||||
|
||||
Shows the content of the [query cache](../query-cache.md).
|
||||
|
||||
Columns:
|
||||
|
||||
- `query` ([String](../../sql-reference/data-types/string.md)) — Query string.
|
||||
- `result_size` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Size of the query cache entry.
|
||||
- `stale` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is stale.
|
||||
- `shared` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is shared between multiple users.
|
||||
- `compressed` ([UInt8](../../sql-reference/data-types/int-uint.md)) — If the query cache entry is compressed.
|
||||
- `expires_at` ([DateTime](../../sql-reference/data-types/datetime.md)) — When the query cache entry becomes stale.
|
||||
- `key_hash` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — A hash of the query string, used as a key to find query cache entries.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.query_cache FORMAT Vertical;
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
query: SELECT 1 SETTINGS use_query_cache = 1
|
||||
result_size: 128
|
||||
stale: 0
|
||||
shared: 0
|
||||
compressed: 1
|
||||
expires_at: 2023-10-13 13:35:45
|
||||
key_hash: 12188185624808016954
|
||||
|
||||
1 row in set. Elapsed: 0.004 sec.
|
||||
```
|
||||
@ -55,6 +55,7 @@ keeper foo bar
|
||||
- `rmr <path>` -- Recursively deletes path. Confirmation required
|
||||
- `flwc <command>` -- Executes four-letter-word command
|
||||
- `help` -- Prints this message
|
||||
- `get_all_children_number [path]` -- Get all numbers of children nodes under a specific path
|
||||
- `get_stat [path]` -- Returns the node's stat (default `.`)
|
||||
- `find_super_nodes <threshold> [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`)
|
||||
- `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive
|
||||
|
||||
@ -104,7 +104,7 @@ SELECT argMin((a, b), (b, a)), min(tuple(b, a)) FROM test;
|
||||
└──────────────────────────────────┴──────────────────┘
|
||||
|
||||
SELECT argMin(a, tuple(b)) FROM test;
|
||||
┌─argMax(a, tuple(b))─┐
|
||||
┌─argMin(a, tuple(b))─┐
|
||||
│ d │ -- `Tuple` can be used in `min` to not skip rows with `NULL` values as b.
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
@ -73,7 +73,7 @@ Calculates the product of two values `a` and `b`.
|
||||
multiply(a, b)
|
||||
```
|
||||
|
||||
Alias: `a \* b` (operator)
|
||||
Alias: `a * b` (operator)
|
||||
|
||||
## divide
|
||||
|
||||
@ -345,9 +345,9 @@ Result:
|
||||
┌─multiply(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
|
||||
│ -26.8609633 │
|
||||
└───────────────────────────────────────────────────────────┘
|
||||
┌─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
|
||||
│ -26.8609 │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
┌───────a─┬──────b─┬─multiplyDecimal(toDecimal64(-12.647, 3), toDecimal32(2.1239, 4))─┐
|
||||
│ -12.647 │ 2.1239 │ -26.8609 │
|
||||
└─────────┴────────┴──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
@ -441,3 +441,40 @@ DB::Exception: Decimal result's scale is less than argument's one: While process
|
||||
│ -12 │ 2.1 │ -5.7 │ -5.71428 │
|
||||
└─────┴─────┴────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## byteSwap
|
||||
|
||||
Reverses the bytes of an integer, i.e. changes its [endianness](https://en.wikipedia.org/wiki/Endianness).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
byteSwap(a)
|
||||
```
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
byteSwap(3351772109)
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```result
|
||||
┌─byteSwap(3351772109)─┐
|
||||
│ 3455829959 │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
The above example can be worked out in the following manner:
|
||||
1. Convert the base-10 integer to its equivalent hexadecimal format in big-endian format, i.e. 3351772109 -> C7 C7 FB CD (4 bytes)
|
||||
2. Reverse the bytes, i.e. C7 C7 FB CD -> CD FB C7 C7
|
||||
3. Convert the result back to an integer assuming big-endian, i.e. CD FB C7 C7 -> 3455829959
|
||||
|
||||
One use case of this function is reversing IPv4s:
|
||||
|
||||
```result
|
||||
┌─toIPv4(byteSwap(toUInt32(toIPv4('205.251.199.199'))))─┐
|
||||
│ 199.199.251.205 │
|
||||
└───────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -204,6 +204,20 @@ Alias: `timezone`.
|
||||
|
||||
Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT timezone()
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─timezone()─────┐
|
||||
│ America/Denver │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
**See also**
|
||||
|
||||
- [serverTimeZone](#serverTimeZone)
|
||||
@ -227,6 +241,20 @@ Alias: `serverTimezone`.
|
||||
|
||||
Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT serverTimeZone()
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─serverTimeZone()─┐
|
||||
│ UTC │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
**See also**
|
||||
|
||||
- [timeZone](#timeZone)
|
||||
@ -366,37 +394,189 @@ Result:
|
||||
|
||||
## toYear
|
||||
|
||||
Converts a date or date with time to the year number (AD) as UInt16 value.
|
||||
Converts a date or date with time to the year number (AD) as `UInt16` value.
|
||||
|
||||
Alias: `YEAR`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toYear(value)
|
||||
```
|
||||
|
||||
Alias: `YEAR`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The year of the given date/time
|
||||
|
||||
Type: `UInt16`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toYear(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toYear(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023 │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toQuarter
|
||||
|
||||
Converts a date or date with time to the quarter number as UInt8 value.
|
||||
Converts a date or date with time to the quarter number (1-4) as `UInt8` value.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toQuarter(value)
|
||||
```
|
||||
|
||||
Alias: `QUARTER`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The quarter of the year (1, 2, 3 or 4) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toQuarter(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toQuarter(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2 │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Alias: `QUARTER`.
|
||||
|
||||
## toMonth
|
||||
|
||||
Converts a date or date with time to the month number (1-12) as UInt8 value.
|
||||
Converts a date or date with time to the month number (1-12) as `UInt8` value.
|
||||
|
||||
Alias: `MONTH`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toMonth(value)
|
||||
```
|
||||
|
||||
Alias: `MONTH`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The month of the year (1 - 12) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toMonth(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toMonth(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 4 │
|
||||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toDayOfYear
|
||||
|
||||
Converts a date or date with time to the number of the day of the year (1-366) as UInt16 value.
|
||||
Converts a date or date with time to the number of the day of the year (1-366) as `UInt16` value.
|
||||
|
||||
Alias: `DAYOFYEAR`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toDayOfYear(value)
|
||||
```
|
||||
|
||||
Alias: `DAYOFYEAR`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The day of the year (1 - 366) of the given date/time
|
||||
|
||||
Type: `UInt16`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toDayOfYear(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toDayOfYear(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 111 │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toDayOfMonth
|
||||
|
||||
Converts a date or date with time to the number of the day in the month (1-31) as UInt8 value.
|
||||
Converts a date or date with time to the number of the day in the month (1-31) as `UInt8` value.
|
||||
|
||||
Aliases: `DAYOFMONTH`, `DAY`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toDayOfMonth(value)
|
||||
```
|
||||
|
||||
Aliases: `DAYOFMONTH`, `DAY`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The day of the month (1 - 31) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toDayOfMonth(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toDayOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 21 │
|
||||
└─────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toDayOfWeek
|
||||
|
||||
Converts a date or date with time to the number of the day in the week as UInt8 value.
|
||||
Converts a date or date with time to the number of the day in the week as `UInt8` value.
|
||||
|
||||
The two-argument form of `toDayOfWeek()` enables you to specify whether the week starts on Monday or Sunday, and whether the return value should be in the range from 0 to 6 or 1 to 7. If the mode argument is omitted, the default mode is 0. The time zone of the date can be specified as the third argument.
|
||||
|
||||
@ -407,33 +587,153 @@ The two-argument form of `toDayOfWeek()` enables you to specify whether the week
|
||||
| 2 | Sunday | 0-6: Sunday = 0, Monday = 1, ..., Saturday = 6 |
|
||||
| 3 | Sunday | 1-7: Sunday = 1, Monday = 2, ..., Saturday = 7 |
|
||||
|
||||
Alias: `DAYOFWEEK`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
toDayOfWeek(t[, mode[, timezone]])
|
||||
```
|
||||
|
||||
Alias: `DAYOFWEEK`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
- `mode` - determines what the first day of the week is. Possible values are 0, 1, 2 or 3. See the table above for the differences.
|
||||
- `timezone` - optional parameter, it behaves like any other conversion function
|
||||
|
||||
The first argument can also be specified as [String](../data-types/string.md) in a format supported by [parseDateTime64BestEffort()](type-conversion-functions.md#parsedatetime64besteffort). Support for string arguments exists only for reasons of compatibility with MySQL which is expected by certain 3rd party tools. As string argument support may in future be made dependent on new MySQL-compatibility settings and because string parsing is generally slow, it is recommended to not use it.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The day of the month (1 - 31) of the given date/time
|
||||
|
||||
**Example**
|
||||
|
||||
The following date is April 21, 2023, which was a Friday:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toDayOfWeek(toDateTime('2023-04-21')),
|
||||
toDayOfWeek(toDateTime('2023-04-21'), 1)
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toDayOfWeek(toDateTime('2023-04-21'))─┬─toDayOfWeek(toDateTime('2023-04-21'), 1)─┐
|
||||
│ 5 │ 4 │
|
||||
└───────────────────────────────────────┴──────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toHour
|
||||
|
||||
Converts a date with time the number of the hour in 24-hour time (0-23) as UInt8 value.
|
||||
Converts a date with time to the number of the hour in 24-hour time (0-23) as `UInt8` value.
|
||||
|
||||
Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always true – even in Moscow the clocks were twice changed at a different time).
|
||||
Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always exactly when it occurs - it depends on the timezone).
|
||||
|
||||
Alias: `HOUR`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toHour(value)
|
||||
```
|
||||
|
||||
Alias: `HOUR`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The hour of the day (0 - 23) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toHour(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toHour(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 10 │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toMinute
|
||||
|
||||
Converts a date with time to the number of the minute of the hour (0-59) as UInt8 value.
|
||||
Converts a date with time to the number of the minute of the hour (0-59) as `UInt8` value.
|
||||
|
||||
Alias: `MINUTE`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toMinute(value)
|
||||
```
|
||||
|
||||
Alias: `MINUTE`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The minute of the hour (0 - 59) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toMinute(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toMinute(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 20 │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toSecond
|
||||
|
||||
Converts a date with time to the second in the minute (0-59) as UInt8 value. Leap seconds are not considered.
|
||||
Converts a date with time to the second in the minute (0-59) as `UInt8` value. Leap seconds are not considered.
|
||||
|
||||
Alias: `SECOND`.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toSecond(value)
|
||||
```
|
||||
|
||||
Alias: `SECOND`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The second in the minute (0 - 59) of the given date/time
|
||||
|
||||
Type: `UInt8`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toSecond(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toSecond(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 30 │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toUnixTimestamp
|
||||
|
||||
@ -496,48 +796,220 @@ Behavior for
|
||||
|
||||
## toStartOfYear
|
||||
|
||||
Rounds down a date or date with time to the first day of the year.
|
||||
Returns the date.
|
||||
Rounds down a date or date with time to the first day of the year. Returns the date as a `Date` object.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfYear(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The first day of the year of the input date/time
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toStartOfYear(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfYear(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-01-01 │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfISOYear
|
||||
|
||||
Rounds down a date or date with time to the first day of ISO year.
|
||||
Returns the date.
|
||||
Rounds down a date or date with time to the first day of the ISO year, which can be different than a "regular" year. (See [https://en.wikipedia.org/wiki/ISO_week_date](https://en.wikipedia.org/wiki/ISO_week_date).)
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfISOYear(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The first day of the year of the input date/time
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toStartOfISOYear(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfISOYear(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-01-02 │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfQuarter
|
||||
|
||||
Rounds down a date or date with time to the first day of the quarter.
|
||||
The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October.
|
||||
Rounds down a date or date with time to the first day of the quarter. The first day of the quarter is either 1 January, 1 April, 1 July, or 1 October.
|
||||
Returns the date.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfQuarter(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The first day of the quarter of the given date/time
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toStartOfQuarter(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfQuarter(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-04-01 │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfMonth
|
||||
|
||||
Rounds down a date or date with time to the first day of the month.
|
||||
Returns the date.
|
||||
Rounds down a date or date with time to the first day of the month. Returns the date.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfMonth(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The first day of the month of the given date/time
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toStartOfMonth(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-04-01 │
|
||||
└───────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
:::note
|
||||
The behavior of parsing incorrect dates is implementation specific. ClickHouse may return zero date, throw an exception or do “natural” overflow.
|
||||
The behavior of parsing incorrect dates is implementation specific. ClickHouse may return zero date, throw an exception, or do “natural” overflow.
|
||||
:::
|
||||
|
||||
## toLastDayOfMonth
|
||||
|
||||
Rounds a date, or date with time, to the last day of the month.
|
||||
Returns the date.
|
||||
Rounds a date or date with time to the last day of the month. Returns the date.
|
||||
|
||||
Alias: `LAST_DAY`.
|
||||
**Syntax**
|
||||
|
||||
If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead.
|
||||
```sql
|
||||
toLastDayOfMonth(value)
|
||||
```
|
||||
|
||||
Alias: `LAST_DAY`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The last day of the month of the given date/time
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toLastDayOfMonth(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toLastDayOfMonth(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-04-30 │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toMonday
|
||||
|
||||
Rounds down a date, or date with time, to the nearest Monday.
|
||||
Returns the date.
|
||||
Rounds down a date or date with time to the nearest Monday. Returns the date.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toMonday(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The date of the nearest Monday on or prior to the given date
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toMonday(toDateTime('2023-04-21 10:20:30')), /* a Friday */
|
||||
toMonday(toDate('2023-04-24')), /* already a Monday */
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toMonday(toDateTime('2023-04-21 10:20:30'))─┬─toMonday(toDate('2023-04-24'))─┐
|
||||
│ 2023-04-17 │ 2023-04-24 │
|
||||
└─────────────────────────────────────────────┴────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfWeek
|
||||
|
||||
Rounds a date or date with time down to the nearest Sunday or Monday.
|
||||
Returns the date.
|
||||
The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, mode is assumed as 0.
|
||||
Rounds a date or date with time down to the nearest Sunday or Monday. Returns the date. The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, it defaults to 0.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -545,10 +1017,43 @@ The mode argument works exactly like the mode argument in function `toWeek()`. I
|
||||
toStartOfWeek(t[, mode[, timezone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
- `mode` - determines the first day of the week as described in the [toWeek()](date-time-functions#toweek) function
|
||||
- `timezone` - Optional parameter, it behaves like any other conversion function
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The date of the nearest Sunday or Monday on or prior to the given date, depending on the mode
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfWeek(toDateTime('2023-04-21 10:20:30')), /* a Friday */
|
||||
toStartOfWeek(toDateTime('2023-04-21 10:20:30'), 1), /* a Friday */
|
||||
toStartOfWeek(toDate('2023-04-24')), /* a Monday */
|
||||
toStartOfWeek(toDate('2023-04-24'), 1) /* a Monday */
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toStartOfWeek(toDateTime('2023-04-21 10:20:30')): 2023-04-16
|
||||
toStartOfWeek(toDateTime('2023-04-21 10:20:30'), 1): 2023-04-17
|
||||
toStartOfWeek(toDate('2023-04-24')): 2023-04-23
|
||||
toStartOfWeek(toDate('2023-04-24'), 1): 2023-04-24
|
||||
```
|
||||
|
||||
## toLastDayOfWeek
|
||||
|
||||
Rounds a date or date with time up to the nearest Saturday or Sunday.
|
||||
Returns the date.
|
||||
Rounds a date or date with time up to the nearest Saturday or Sunday. Returns the date.
|
||||
The mode argument works exactly like the mode argument in function `toWeek()`. If no mode is specified, mode is assumed as 0.
|
||||
|
||||
**Syntax**
|
||||
@ -557,18 +1062,148 @@ The mode argument works exactly like the mode argument in function `toWeek()`. I
|
||||
toLastDayOfWeek(t[, mode[, timezone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `t` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
- `mode` - determines the last day of the week as described in the [toWeek()](date-time-functions#toweek) function
|
||||
- `timezone` - Optional parameter, it behaves like any other conversion function
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The date of the nearest Sunday or Monday on or after the given date, depending on the mode
|
||||
|
||||
Type: `Date`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toLastDayOfWeek(toDateTime('2023-04-21 10:20:30')), /* a Friday */
|
||||
toLastDayOfWeek(toDateTime('2023-04-21 10:20:30'), 1), /* a Friday */
|
||||
toLastDayOfWeek(toDate('2023-04-22')), /* a Saturday */
|
||||
toLastDayOfWeek(toDate('2023-04-22'), 1) /* a Saturday */
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toLastDayOfWeek(toDateTime('2023-04-21 10:20:30')): 2023-04-22
|
||||
toLastDayOfWeek(toDateTime('2023-04-21 10:20:30'), 1): 2023-04-23
|
||||
toLastDayOfWeek(toDate('2023-04-22')): 2023-04-22
|
||||
toLastDayOfWeek(toDate('2023-04-22'), 1): 2023-04-23
|
||||
```
|
||||
|
||||
## toStartOfDay
|
||||
|
||||
Rounds down a date with time to the start of the day.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfDay(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the day of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT toStartOfDay(toDateTime('2023-04-21 10:20:30'))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfDay(toDateTime('2023-04-21 10:20:30'))─┐
|
||||
│ 2023-04-21 00:00:00 │
|
||||
└─────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfHour
|
||||
|
||||
Rounds down a date with time to the start of the hour.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfHour(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the hour of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfHour(toDateTime('2023-04-21 10:20:30')),
|
||||
toStartOfHour(toDateTime64('2023-04-21', 6))
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─toStartOfHour(toDateTime('2023-04-21 10:20:30'))─┬─toStartOfHour(toDateTime64('2023-04-21', 6))─┐
|
||||
│ 2023-04-21 10:00:00 │ 2023-04-21 00:00:00 │
|
||||
└──────────────────────────────────────────────────┴──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toStartOfMinute
|
||||
|
||||
Rounds down a date with time to the start of the minute.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfMinute(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the minute of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfMinute(toDateTime('2023-04-21 10:20:30')),
|
||||
toStartOfMinute(toDateTime64('2023-04-21 10:20:30.5300', 8))
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toStartOfMinute(toDateTime('2023-04-21 10:20:30')): 2023-04-21 10:20:00
|
||||
toStartOfMinute(toDateTime64('2023-04-21 10:20:30.5300', 8)): 2023-04-21 10:20:00
|
||||
```
|
||||
|
||||
## toStartOfSecond
|
||||
|
||||
Truncates sub-seconds.
|
||||
@ -630,14 +1265,122 @@ Result:
|
||||
|
||||
Rounds down a date with time to the start of the five-minute interval.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfFiveMinutes(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the five-minute interval of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:17:00')),
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:20:00')),
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:23:00'))
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:15:00
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:20:00
|
||||
toStartOfFiveMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:20:00
|
||||
```
|
||||
|
||||
## toStartOfTenMinutes
|
||||
|
||||
Rounds down a date with time to the start of the ten-minute interval.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfTenMinutes(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the ten-minute interval of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:17:00')),
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:20:00')),
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:23:00'))
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:10:00
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:20:00
|
||||
toStartOfTenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:20:00
|
||||
```
|
||||
|
||||
## toStartOfFifteenMinutes
|
||||
|
||||
Rounds down the date with time to the start of the fifteen-minute interval.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
toStartOfFifteenMinutes(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The start of the fifteen-minute interval of the given date/time
|
||||
|
||||
Type: `DateTime`
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:17:00')),
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')),
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00'))
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:17:00')): 2023-04-21 10:15:00
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:20:00')): 2023-04-21 10:15:00
|
||||
toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00
|
||||
```
|
||||
|
||||
## toStartOfInterval(time_or_data, INTERVAL x unit \[, time_zone\])
|
||||
|
||||
This function generalizes other `toStartOf*()` functions. For example,
|
||||
@ -748,12 +1491,16 @@ For mode values with a meaning of “contains January 1”, the week contains Ja
|
||||
toWeek(t[, mode[, time_zone]])
|
||||
```
|
||||
|
||||
Alias: `WEEK`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `t` – Date or DateTime.
|
||||
- `mode` – Optional parameter, Range of values is \[0,9\], default is 0.
|
||||
- `Timezone` – Optional parameter, it behaves like any other conversion function.
|
||||
|
||||
The first argument can also be specified as [String](../data-types/string.md) in a format supported by [parseDateTime64BestEffort()](type-conversion-functions.md#parsedatetime64besteffort). Support for string arguments exists only for reasons of compatibility with MySQL which is expected by certain 3rd party tools. As string argument support may in future be made dependent on new MySQL-compatibility settings and because string parsing is generally slow, it is recommended to not use it.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
@ -784,6 +1531,10 @@ The week number returned by `toYearWeek()` can be different from what the `toWee
|
||||
toYearWeek(t[, mode[, timezone]])
|
||||
```
|
||||
|
||||
Alias: `YEARWEEK`
|
||||
|
||||
The first argument can also be specified as [String](../data-types/string.md) in a format supported by [parseDateTime64BestEffort()](type-conversion-functions.md#parsedatetime64besteffort). Support for string arguments exists only for reasons of compatibility with MySQL which is expected by certain 3rd party tools. As string argument support may in future be made dependent on new MySQL-compatibility settings and because string parsing is generally slow, it is recommended to not use it.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
|
||||
@ -68,6 +68,45 @@ WHERE macro = 'test';
|
||||
└───────┴──────────────┘
|
||||
```
|
||||
|
||||
## getHttpHeader
|
||||
Returns the value of specified http header.If there is no such header or the request method is not http, it will return empty string.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
getHttpHeader(name);
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `name` — Http header name .[String](../../sql-reference/data-types/string.md#string)
|
||||
|
||||
**Returned value**
|
||||
|
||||
Value of the specified header.
|
||||
Type:[String](../../sql-reference/data-types/string.md#string).
|
||||
|
||||
|
||||
When we use `clickhouse-client` to execute this function, we'll always get empty string, because client doesn't use http protocol.
|
||||
```sql
|
||||
SELECT getHttpHeader('test')
|
||||
```
|
||||
result:
|
||||
|
||||
```text
|
||||
┌─getHttpHeader('test')─┐
|
||||
│ │
|
||||
└───────────────────────┘
|
||||
```
|
||||
Try to use http request:
|
||||
```shell
|
||||
echo "select getHttpHeader('X-Clickhouse-User')" | curl -H 'X-ClickHouse-User: default' -H 'X-ClickHouse-Key: ' 'http://localhost:8123/' -d @-
|
||||
|
||||
#result
|
||||
default
|
||||
```
|
||||
|
||||
|
||||
## FQDN
|
||||
|
||||
Returns the fully qualified domain name of the ClickHouse server.
|
||||
|
||||
@ -487,7 +487,7 @@ ClickHouse supports temporary tables which have the following characteristics:
|
||||
- The DB can’t be specified for a temporary table. It is created outside of databases.
|
||||
- Impossible to create a temporary table with distributed DDL query on all cluster servers (by using `ON CLUSTER`): this table exists only in the current session.
|
||||
- If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used.
|
||||
- For distributed query processing, temporary tables used in a query are passed to remote servers.
|
||||
- For distributed query processing, temporary tables with Memory engine used in a query are passed to remote servers.
|
||||
|
||||
To create a temporary table, use the following syntax:
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ The `RENAME` query is supported by the [Atomic](../../engines/database-engines/a
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
RENAME DATABASE|TABLE|DICTIONARY name TO new_name [,...] [ON CLUSTER cluster]
|
||||
RENAME [DATABASE|TABLE|DICTIONARY] name TO new_name [,...] [ON CLUSTER cluster]
|
||||
```
|
||||
|
||||
## RENAME DATABASE
|
||||
@ -48,6 +48,11 @@ RENAME TABLE [db1.]name1 TO [db2.]name2 [,...] [ON CLUSTER cluster]
|
||||
RENAME TABLE table_A TO table_A_bak, table_B TO table_B_bak;
|
||||
```
|
||||
|
||||
And you can use a simpler sql:
|
||||
```sql
|
||||
RENAME table_A TO table_A_bak, table_B TO table_B_bak;
|
||||
```
|
||||
|
||||
## RENAME DICTIONARY
|
||||
|
||||
Renames one or several dictionaries. This query can be used to move dictionaries between databases.
|
||||
|
||||
@ -20,7 +20,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
|
||||
...
|
||||
) ENGINE = ReplacingMergeTree([ver])
|
||||
) ENGINE = ReplacingMergeTree([ver [, is_deleted]])
|
||||
[PARTITION BY expr]
|
||||
[ORDER BY expr]
|
||||
[SAMPLE BY expr]
|
||||
@ -86,6 +86,59 @@ SELECT * FROM mySecondReplacingMT FINAL;
|
||||
│ 1 │ first │ 2020-01-01 01:01:01 │
|
||||
└─────┴─────────┴─────────────────────┘
|
||||
```
|
||||
### is_deleted
|
||||
|
||||
`is_deleted` — Имя столбца, который используется во время слияния для обозначения того, нужно ли отображать строку или она подлежит удалению; `1` - для удаления строки, `0` - для отображения строки.
|
||||
|
||||
Тип данных столбца — `UInt8`.
|
||||
|
||||
:::note
|
||||
`is_deleted` может быть использован, если `ver` используется.
|
||||
|
||||
Строка удаляется в следующих случаях:
|
||||
|
||||
- при использовании инструкции `OPTIMIZE ... FINAL CLEANUP`
|
||||
- при использовании инструкции `OPTIMIZE ... FINAL`
|
||||
- параметр движка `clean_deleted_rows` установлен в значение `Always` (по умолчанию - `Never`)
|
||||
- есть новые версии строки
|
||||
|
||||
Не рекомендуется выполнять `FINAL CLEANUP` или использовать параметр движка `clean_deleted_rows` со значением `Always`, это может привести к неожиданным результатам, например удаленные строки могут вновь появиться.
|
||||
|
||||
Вне зависимости от производимых изменений над данными, версия должна увеличиваться. Если у двух строк одна и та же версия, то остается только последняя вставленная строка.
|
||||
:::
|
||||
|
||||
Пример:
|
||||
|
||||
```sql
|
||||
-- with ver and is_deleted
|
||||
CREATE OR REPLACE TABLE myThirdReplacingMT
|
||||
(
|
||||
`key` Int64,
|
||||
`someCol` String,
|
||||
`eventTime` DateTime,
|
||||
`is_deleted` UInt8
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(eventTime, is_deleted)
|
||||
ORDER BY key;
|
||||
|
||||
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 0);
|
||||
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 1);
|
||||
|
||||
select * from myThirdReplacingMT final;
|
||||
|
||||
0 rows in set. Elapsed: 0.003 sec.
|
||||
|
||||
-- delete rows with is_deleted
|
||||
OPTIMIZE TABLE myThirdReplacingMT FINAL CLEANUP;
|
||||
|
||||
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 00:00:00', 0);
|
||||
|
||||
select * from myThirdReplacingMT final;
|
||||
|
||||
┌─key─┬─someCol─┬───────────eventTime─┬─is_deleted─┐
|
||||
│ 1 │ first │ 2020-01-01 00:00:00 │ 0 │
|
||||
└─────┴─────────┴─────────────────────┴────────────┘
|
||||
```
|
||||
|
||||
## Секции запроса
|
||||
|
||||
|
||||
@ -39,6 +39,8 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试
|
||||
- [nestjs-clickhouse](https://github.com/depyronick/nestjs-clickhouse)
|
||||
- [clickhouse-client](https://github.com/depyronick/clickhouse-client)
|
||||
- [node-clickhouse-orm](https://github.com/zimv/node-clickhouse-orm)
|
||||
- [clickhouse-ts](https://github.com/bytadaniel/clickhouse-ts)
|
||||
- [clickcache](https://github.com/bytadaniel/clickcache)
|
||||
- Perl
|
||||
- [perl-DBD-ClickHouse](https://github.com/elcamlost/perl-DBD-ClickHouse)
|
||||
- [HTTP-ClickHouse](https://metacpan.org/release/HTTP-ClickHouse)
|
||||
|
||||
@ -706,6 +706,17 @@ bool Client::processWithFuzzing(const String & full_query)
|
||||
return true;
|
||||
}
|
||||
|
||||
// Kusto is not a subject for fuzzing (yet)
|
||||
if (global_context->getSettingsRef().dialect == DB::Dialect::kusto)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (auto *q = orig_ast->as<ASTSetQuery>())
|
||||
{
|
||||
if (auto *setDialect = q->changes.tryGet("dialect"); setDialect && setDialect->safeGet<String>() == "kusto")
|
||||
return true;
|
||||
}
|
||||
|
||||
// Don't repeat:
|
||||
// - INSERT -- Because the tables may grow too big.
|
||||
// - CREATE -- Because first we run the unmodified query, it will succeed,
|
||||
@ -1253,6 +1264,15 @@ void Client::processConfig()
|
||||
global_context->setQueryKindInitial();
|
||||
global_context->setQuotaClientKey(config().getString("quota_key", ""));
|
||||
global_context->setQueryKind(query_kind);
|
||||
|
||||
if (is_multiquery && !global_context->getSettingsRef().input_format_values_allow_data_after_semicolon.changed)
|
||||
{
|
||||
Settings settings = global_context->getSettings();
|
||||
settings.input_format_values_allow_data_after_semicolon = true;
|
||||
/// Do not send it to the server
|
||||
settings.input_format_values_allow_data_after_semicolon.changed = false;
|
||||
global_context->setSettings(settings);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -269,7 +269,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std
|
||||
res.hostname_difference = std::numeric_limits<size_t>::max();
|
||||
for (const auto & replica : replicas)
|
||||
{
|
||||
size_t difference = getHostNameDifference(local_hostname, replica.host_name);
|
||||
size_t difference = getHostNamePrefixDistance(local_hostname, replica.host_name);
|
||||
res.hostname_difference = std::min(difference, res.hostname_difference);
|
||||
}
|
||||
|
||||
|
||||
@ -324,8 +324,8 @@ void FindBigFamily::execute(const ASTKeeperQuery * query, KeeperClient * client)
|
||||
queue.pop();
|
||||
|
||||
auto children = client->zookeeper->getChildren(next_path);
|
||||
std::transform(children.cbegin(), children.cend(), children.begin(), [&](const String & child) { return next_path / child; });
|
||||
|
||||
for (auto & child : children)
|
||||
child = next_path / child;
|
||||
auto response = client->zookeeper->get(children);
|
||||
|
||||
for (size_t i = 0; i < response.size(); ++i)
|
||||
@ -475,4 +475,45 @@ void FourLetterWordCommand::execute(const ASTKeeperQuery * query, KeeperClient *
|
||||
std::cout << client->executeFourLetterCommand(query->args[0].safeGet<String>()) << "\n";
|
||||
}
|
||||
|
||||
bool GetAllChildrenNumberCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const
|
||||
{
|
||||
String path;
|
||||
if (!parseKeeperPath(pos, expected, path))
|
||||
path = ".";
|
||||
|
||||
node->args.push_back(std::move(path));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void GetAllChildrenNumberCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const
|
||||
{
|
||||
auto path = client->getAbsolutePath(query->args[0].safeGet<String>());
|
||||
|
||||
std::queue<fs::path> queue;
|
||||
queue.push(path);
|
||||
Coordination::Stat stat;
|
||||
client->zookeeper->get(path, &stat);
|
||||
|
||||
int totalNumChildren = stat.numChildren;
|
||||
while (!queue.empty())
|
||||
{
|
||||
auto next_path = queue.front();
|
||||
queue.pop();
|
||||
|
||||
auto children = client->zookeeper->getChildren(next_path);
|
||||
for (auto & child : children)
|
||||
child = next_path / child;
|
||||
auto response = client->zookeeper->get(children);
|
||||
|
||||
for (size_t i = 0; i < response.size(); ++i)
|
||||
{
|
||||
totalNumChildren += response[i].stat.numChildren;
|
||||
queue.push(children[i]);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << totalNumChildren << "\n";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -238,4 +238,18 @@ class FourLetterWordCommand : public IKeeperClientCommand
|
||||
String getHelpMessage() const override { return "{} <command> -- Executes four-letter-word command"; }
|
||||
};
|
||||
|
||||
class GetAllChildrenNumberCommand : public IKeeperClientCommand
|
||||
{
|
||||
String getName() const override { return "get_all_children_number"; }
|
||||
|
||||
bool parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, Expected & expected) const override;
|
||||
|
||||
void execute(const ASTKeeperQuery * query, KeeperClient * client) const override;
|
||||
|
||||
String getHelpMessage() const override
|
||||
{
|
||||
return "{} [path] -- Get all numbers of children nodes under a specific path";
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include "Commands.h"
|
||||
#include <Client/ReplxxLineReader.h>
|
||||
#include <Client/ClientBase.h>
|
||||
#include "Common/VersionNumber.h"
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/EventNotifier.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
@ -206,6 +207,7 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */)
|
||||
std::make_shared<SyncCommand>(),
|
||||
std::make_shared<HelpCommand>(),
|
||||
std::make_shared<FourLetterWordCommand>(),
|
||||
std::make_shared<GetAllChildrenNumberCommand>(),
|
||||
});
|
||||
|
||||
String home_path;
|
||||
|
||||
@ -783,6 +783,15 @@ void LocalServer::processConfig()
|
||||
|
||||
global_context->setQueryKindInitial();
|
||||
global_context->setQueryKind(query_kind);
|
||||
|
||||
if (is_multiquery && !global_context->getSettingsRef().input_format_values_allow_data_after_semicolon.changed)
|
||||
{
|
||||
Settings settings = global_context->getSettings();
|
||||
settings.input_format_values_allow_data_after_semicolon = true;
|
||||
/// Do not send it to the server
|
||||
settings.input_format_values_allow_data_after_semicolon.changed = false;
|
||||
global_context->setSettings(settings);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1474,7 +1474,7 @@ try
|
||||
|
||||
{
|
||||
std::lock_guard lock(servers_lock);
|
||||
/// We should start interserver communications before (and more imporant shutdown after) tables.
|
||||
/// We should start interserver communications before (and more important shutdown after) tables.
|
||||
/// Because server can wait for a long-running queries (for example in tcp_handler) after interserver handler was already shut down.
|
||||
/// In this case we will have replicated tables which are unable to send any parts to other replicas, but still can
|
||||
/// communicate with zookeeper, execute merges, etc.
|
||||
|
||||
@ -420,6 +420,10 @@
|
||||
color: var(--auth-error-color);
|
||||
}
|
||||
|
||||
#charts > div:only-child .display-only-if-more-than-one-chart {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Source: https://cdn.jsdelivr.net/npm/uplot@1.6.21/dist/uPlot.min.css
|
||||
* It is copy-pasted to lower the number of requests.
|
||||
*/
|
||||
@ -931,6 +935,9 @@ function insertChart(i) {
|
||||
saveState();
|
||||
});
|
||||
|
||||
move.classList.add('display-only-if-more-than-one-chart');
|
||||
maximize.classList.add('display-only-if-more-than-one-chart');
|
||||
|
||||
edit_buttons.appendChild(move);
|
||||
edit_buttons.appendChild(maximize);
|
||||
edit_buttons.appendChild(edit);
|
||||
|
||||
@ -9,6 +9,7 @@ profiles:
|
||||
# random - choose random replica from set of replicas with minimum number of errors
|
||||
# nearest_hostname - from set of replicas with minimum number of errors, choose replica
|
||||
# with minimum number of different symbols between replica's hostname and local hostname (Hamming distance).
|
||||
# hostname_levenshtein_distance - just the same with nearest_hostname but calculate the difference by Levenshtein distance.
|
||||
# in_order - first live replica is chosen in specified order.
|
||||
# first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
|
||||
load_balancing: random
|
||||
|
||||
@ -1264,7 +1264,8 @@ private:
|
||||
size_t identifier_bind_size,
|
||||
const QueryTreeNodePtr & compound_expression,
|
||||
String compound_expression_source,
|
||||
IdentifierResolveScope & scope);
|
||||
IdentifierResolveScope & scope,
|
||||
bool can_be_not_found = false);
|
||||
|
||||
QueryTreeNodePtr tryResolveIdentifierFromExpressionArguments(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope);
|
||||
|
||||
@ -1313,6 +1314,14 @@ private:
|
||||
IdentifierResolveScope & scope,
|
||||
IdentifierResolveSettings identifier_resolve_settings = {});
|
||||
|
||||
QueryTreeNodePtr tryResolveIdentifierFromStorage(
|
||||
const Identifier & identifier,
|
||||
const QueryTreeNodePtr & table_expression_node,
|
||||
const TableExpressionData & table_expression_data,
|
||||
IdentifierResolveScope & scope,
|
||||
size_t identifier_column_qualifier_parts,
|
||||
bool can_be_not_found = false);
|
||||
|
||||
/// Resolve query tree nodes functions
|
||||
|
||||
void qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & column_nodes,
|
||||
@ -2395,11 +2404,13 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveTableIdentifierFromDatabaseCatalog(con
|
||||
}
|
||||
|
||||
/// Resolve identifier from compound expression
|
||||
/// If identifier cannot be resolved throw exception or return nullptr if can_be_not_found is true
|
||||
QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromCompoundExpression(const Identifier & expression_identifier,
|
||||
size_t identifier_bind_size,
|
||||
const QueryTreeNodePtr & compound_expression,
|
||||
String compound_expression_source,
|
||||
IdentifierResolveScope & scope)
|
||||
IdentifierResolveScope & scope,
|
||||
bool can_be_not_found)
|
||||
{
|
||||
Identifier compound_expression_identifier;
|
||||
for (size_t i = 0; i < identifier_bind_size; ++i)
|
||||
@ -2412,6 +2423,23 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromCompoundExpression(const
|
||||
|
||||
if (!expression_type->hasSubcolumn(nested_path.getFullName()))
|
||||
{
|
||||
if (auto * column = compound_expression->as<ColumnNode>())
|
||||
{
|
||||
const DataTypePtr & column_type = column->getColumn().getTypeInStorage();
|
||||
if (column_type->getTypeId() == TypeIndex::Object)
|
||||
{
|
||||
const auto * object_type = checkAndGetDataType<DataTypeObject>(column_type.get());
|
||||
if (object_type->getSchemaFormat() == "json" && object_type->hasNullableSubcolumns())
|
||||
{
|
||||
QueryTreeNodePtr constant_node_null = std::make_shared<ConstantNode>(Field());
|
||||
return constant_node_null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (can_be_not_found)
|
||||
return {};
|
||||
|
||||
std::unordered_set<Identifier> valid_identifiers;
|
||||
collectCompoundExpressionValidIdentifiersForTypoCorrection(expression_identifier,
|
||||
expression_type,
|
||||
@ -2427,20 +2455,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromCompoundExpression(const
|
||||
compound_expression_from_error_message += compound_expression_source;
|
||||
}
|
||||
|
||||
if (auto * column = compound_expression->as<ColumnNode>())
|
||||
{
|
||||
const DataTypePtr & column_type = column->getColumn().getTypeInStorage();
|
||||
if (column_type->getTypeId() == TypeIndex::Object)
|
||||
{
|
||||
const auto * object_type = checkAndGetDataType<DataTypeObject>(column_type.get());
|
||||
if (object_type->getSchemaFormat() == "json" && object_type->hasNullableSubcolumns())
|
||||
{
|
||||
QueryTreeNodePtr constant_node_null = std::make_shared<ConstantNode>(Field());
|
||||
return constant_node_null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
|
||||
"Identifier {} nested path {} cannot be resolved from type {}{}. In scope {}{}",
|
||||
expression_identifier,
|
||||
@ -2796,6 +2810,160 @@ bool QueryAnalyzer::tryBindIdentifierToTableExpressions(const IdentifierLookup &
|
||||
return can_bind_identifier_to_table_expression;
|
||||
}
|
||||
|
||||
QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromStorage(
|
||||
const Identifier & identifier,
|
||||
const QueryTreeNodePtr & table_expression_node,
|
||||
const TableExpressionData & table_expression_data,
|
||||
IdentifierResolveScope & scope,
|
||||
size_t identifier_column_qualifier_parts,
|
||||
bool can_be_not_found)
|
||||
{
|
||||
auto identifier_without_column_qualifier = identifier;
|
||||
identifier_without_column_qualifier.popFirst(identifier_column_qualifier_parts);
|
||||
|
||||
/** Compound identifier cannot be resolved directly from storage if storage is not table.
|
||||
*
|
||||
* Example: SELECT test_table.id.value1.value2 FROM test_table;
|
||||
* In table storage column test_table.id.value1.value2 will exists.
|
||||
*
|
||||
* Example: SELECT test_subquery.compound_expression.value FROM (SELECT compound_expression AS value) AS test_subquery;
|
||||
* Here there is no column with name test_subquery.compound_expression.value, and additional wrap in tuple element is required.
|
||||
*/
|
||||
|
||||
QueryTreeNodePtr result_expression;
|
||||
bool match_full_identifier = false;
|
||||
|
||||
auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.getFullName());
|
||||
if (it != table_expression_data.column_name_to_column_node.end())
|
||||
{
|
||||
match_full_identifier = true;
|
||||
result_expression = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.at(0));
|
||||
if (it != table_expression_data.column_name_to_column_node.end())
|
||||
result_expression = it->second;
|
||||
}
|
||||
|
||||
bool clone_is_needed = true;
|
||||
|
||||
String table_expression_source = table_expression_data.table_expression_description;
|
||||
if (!table_expression_data.table_expression_name.empty())
|
||||
table_expression_source += " with name " + table_expression_data.table_expression_name;
|
||||
|
||||
if (result_expression && !match_full_identifier && identifier_without_column_qualifier.isCompound())
|
||||
{
|
||||
size_t identifier_bind_size = identifier_column_qualifier_parts + 1;
|
||||
result_expression = tryResolveIdentifierFromCompoundExpression(identifier,
|
||||
identifier_bind_size,
|
||||
result_expression,
|
||||
table_expression_source,
|
||||
scope,
|
||||
can_be_not_found);
|
||||
if (can_be_not_found && !result_expression)
|
||||
return {};
|
||||
clone_is_needed = false;
|
||||
}
|
||||
|
||||
if (!result_expression)
|
||||
{
|
||||
QueryTreeNodes nested_column_nodes;
|
||||
DataTypes nested_types;
|
||||
Array nested_names_array;
|
||||
|
||||
for (const auto & [column_name, _] : table_expression_data.column_names_and_types)
|
||||
{
|
||||
Identifier column_name_identifier_without_last_part(column_name);
|
||||
auto column_name_identifier_last_part = column_name_identifier_without_last_part.getParts().back();
|
||||
column_name_identifier_without_last_part.popLast();
|
||||
|
||||
if (identifier_without_column_qualifier.getFullName() != column_name_identifier_without_last_part.getFullName())
|
||||
continue;
|
||||
|
||||
auto column_node_it = table_expression_data.column_name_to_column_node.find(column_name);
|
||||
if (column_node_it == table_expression_data.column_name_to_column_node.end())
|
||||
continue;
|
||||
|
||||
const auto & column_node = column_node_it->second;
|
||||
const auto & column_type = column_node->getColumnType();
|
||||
const auto * column_type_array = typeid_cast<const DataTypeArray *>(column_type.get());
|
||||
if (!column_type_array)
|
||||
continue;
|
||||
|
||||
nested_column_nodes.push_back(column_node);
|
||||
nested_types.push_back(column_type_array->getNestedType());
|
||||
nested_names_array.push_back(Field(std::move(column_name_identifier_last_part)));
|
||||
}
|
||||
|
||||
if (!nested_types.empty())
|
||||
{
|
||||
auto nested_function_node = std::make_shared<FunctionNode>("nested");
|
||||
auto & nested_function_node_arguments = nested_function_node->getArguments().getNodes();
|
||||
|
||||
auto nested_function_names_array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
|
||||
auto nested_function_names_constant_node = std::make_shared<ConstantNode>(std::move(nested_names_array),
|
||||
std::move(nested_function_names_array_type));
|
||||
nested_function_node_arguments.push_back(std::move(nested_function_names_constant_node));
|
||||
nested_function_node_arguments.insert(nested_function_node_arguments.end(),
|
||||
nested_column_nodes.begin(),
|
||||
nested_column_nodes.end());
|
||||
|
||||
auto nested_function = FunctionFactory::instance().get(nested_function_node->getFunctionName(), scope.context);
|
||||
nested_function_node->resolveAsFunction(nested_function->build(nested_function_node->getArgumentColumns()));
|
||||
|
||||
clone_is_needed = false;
|
||||
result_expression = std::move(nested_function_node);
|
||||
}
|
||||
}
|
||||
|
||||
if (!result_expression)
|
||||
{
|
||||
std::unordered_set<Identifier> valid_identifiers;
|
||||
collectTableExpressionValidIdentifiersForTypoCorrection(identifier,
|
||||
table_expression_node,
|
||||
table_expression_data,
|
||||
valid_identifiers);
|
||||
|
||||
auto hints = collectIdentifierTypoHints(identifier, valid_identifiers);
|
||||
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Identifier '{}' cannot be resolved from {}. In scope {}{}",
|
||||
identifier.getFullName(),
|
||||
table_expression_source,
|
||||
scope.scope_node->formatASTForErrorMessage(),
|
||||
getHintsErrorMessageSuffix(hints));
|
||||
}
|
||||
|
||||
if (clone_is_needed)
|
||||
result_expression = result_expression->clone();
|
||||
|
||||
auto qualified_identifier = identifier;
|
||||
|
||||
for (size_t i = 0; i < identifier_column_qualifier_parts; ++i)
|
||||
{
|
||||
auto qualified_identifier_with_removed_part = qualified_identifier;
|
||||
qualified_identifier_with_removed_part.popFirst();
|
||||
|
||||
if (qualified_identifier_with_removed_part.empty())
|
||||
break;
|
||||
|
||||
IdentifierLookup column_identifier_lookup = {qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION};
|
||||
if (tryBindIdentifierToAliases(column_identifier_lookup, scope))
|
||||
break;
|
||||
|
||||
if (table_expression_data.should_qualify_columns &&
|
||||
tryBindIdentifierToTableExpressions(column_identifier_lookup, table_expression_node, scope))
|
||||
break;
|
||||
|
||||
qualified_identifier = std::move(qualified_identifier_with_removed_part);
|
||||
}
|
||||
|
||||
auto qualified_identifier_full_name = qualified_identifier.getFullName();
|
||||
node_to_projection_name.emplace(result_expression, std::move(qualified_identifier_full_name));
|
||||
|
||||
return result_expression;
|
||||
}
|
||||
|
||||
QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup,
|
||||
const QueryTreeNodePtr & table_expression_node,
|
||||
IdentifierResolveScope & scope)
|
||||
@ -2836,151 +3004,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id
|
||||
return {};
|
||||
}
|
||||
|
||||
auto resolve_identifier_from_storage_or_throw = [&](size_t identifier_column_qualifier_parts) -> QueryTreeNodePtr
|
||||
{
|
||||
auto identifier_without_column_qualifier = identifier;
|
||||
identifier_without_column_qualifier.popFirst(identifier_column_qualifier_parts);
|
||||
|
||||
/** Compound identifier cannot be resolved directly from storage if storage is not table.
|
||||
*
|
||||
* Example: SELECT test_table.id.value1.value2 FROM test_table;
|
||||
* In table storage column test_table.id.value1.value2 will exists.
|
||||
*
|
||||
* Example: SELECT test_subquery.compound_expression.value FROM (SELECT compound_expression AS value) AS test_subquery;
|
||||
* Here there is no column with name test_subquery.compound_expression.value, and additional wrap in tuple element is required.
|
||||
*/
|
||||
|
||||
QueryTreeNodePtr result_expression;
|
||||
bool match_full_identifier = false;
|
||||
|
||||
auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.getFullName());
|
||||
if (it != table_expression_data.column_name_to_column_node.end())
|
||||
{
|
||||
match_full_identifier = true;
|
||||
result_expression = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.at(0));
|
||||
if (it != table_expression_data.column_name_to_column_node.end())
|
||||
result_expression = it->second;
|
||||
}
|
||||
|
||||
bool clone_is_needed = true;
|
||||
|
||||
String table_expression_source = table_expression_data.table_expression_description;
|
||||
if (!table_expression_data.table_expression_name.empty())
|
||||
table_expression_source += " with name " + table_expression_data.table_expression_name;
|
||||
|
||||
if (result_expression && !match_full_identifier && identifier_without_column_qualifier.isCompound())
|
||||
{
|
||||
size_t identifier_bind_size = identifier_column_qualifier_parts + 1;
|
||||
result_expression = tryResolveIdentifierFromCompoundExpression(identifier_lookup.identifier,
|
||||
identifier_bind_size,
|
||||
result_expression,
|
||||
table_expression_source,
|
||||
scope);
|
||||
clone_is_needed = false;
|
||||
}
|
||||
|
||||
if (!result_expression)
|
||||
{
|
||||
QueryTreeNodes nested_column_nodes;
|
||||
DataTypes nested_types;
|
||||
Array nested_names_array;
|
||||
|
||||
for (auto & [column_name, _] : table_expression_data.column_names_and_types)
|
||||
{
|
||||
Identifier column_name_identifier_without_last_part(column_name);
|
||||
auto column_name_identifier_last_part = column_name_identifier_without_last_part.getParts().back();
|
||||
column_name_identifier_without_last_part.popLast();
|
||||
|
||||
if (identifier_without_column_qualifier.getFullName() != column_name_identifier_without_last_part.getFullName())
|
||||
continue;
|
||||
|
||||
auto column_node_it = table_expression_data.column_name_to_column_node.find(column_name);
|
||||
if (column_node_it == table_expression_data.column_name_to_column_node.end())
|
||||
continue;
|
||||
|
||||
const auto & column_node = column_node_it->second;
|
||||
const auto & column_type = column_node->getColumnType();
|
||||
const auto * column_type_array = typeid_cast<const DataTypeArray *>(column_type.get());
|
||||
if (!column_type_array)
|
||||
continue;
|
||||
|
||||
nested_column_nodes.push_back(column_node);
|
||||
nested_types.push_back(column_type_array->getNestedType());
|
||||
nested_names_array.push_back(Field(std::move(column_name_identifier_last_part)));
|
||||
}
|
||||
|
||||
if (!nested_types.empty())
|
||||
{
|
||||
auto nested_function_node = std::make_shared<FunctionNode>("nested");
|
||||
auto & nested_function_node_arguments = nested_function_node->getArguments().getNodes();
|
||||
|
||||
auto nested_function_names_array_type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
|
||||
auto nested_function_names_constant_node = std::make_shared<ConstantNode>(std::move(nested_names_array),
|
||||
std::move(nested_function_names_array_type));
|
||||
nested_function_node_arguments.push_back(std::move(nested_function_names_constant_node));
|
||||
nested_function_node_arguments.insert(nested_function_node_arguments.end(),
|
||||
nested_column_nodes.begin(),
|
||||
nested_column_nodes.end());
|
||||
|
||||
auto nested_function = FunctionFactory::instance().get(nested_function_node->getFunctionName(), scope.context);
|
||||
nested_function_node->resolveAsFunction(nested_function->build(nested_function_node->getArgumentColumns()));
|
||||
|
||||
clone_is_needed = false;
|
||||
result_expression = std::move(nested_function_node);
|
||||
}
|
||||
}
|
||||
|
||||
if (!result_expression)
|
||||
{
|
||||
std::unordered_set<Identifier> valid_identifiers;
|
||||
collectTableExpressionValidIdentifiersForTypoCorrection(identifier,
|
||||
table_expression_node,
|
||||
table_expression_data,
|
||||
valid_identifiers);
|
||||
|
||||
auto hints = collectIdentifierTypoHints(identifier, valid_identifiers);
|
||||
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Identifier '{}' cannot be resolved from {}. In scope {}{}",
|
||||
identifier.getFullName(),
|
||||
table_expression_source,
|
||||
scope.scope_node->formatASTForErrorMessage(),
|
||||
getHintsErrorMessageSuffix(hints));
|
||||
}
|
||||
|
||||
if (clone_is_needed)
|
||||
result_expression = result_expression->clone();
|
||||
|
||||
auto qualified_identifier = identifier;
|
||||
|
||||
for (size_t i = 0; i < identifier_column_qualifier_parts; ++i)
|
||||
{
|
||||
auto qualified_identifier_with_removed_part = qualified_identifier;
|
||||
qualified_identifier_with_removed_part.popFirst();
|
||||
|
||||
if (qualified_identifier_with_removed_part.empty())
|
||||
break;
|
||||
|
||||
IdentifierLookup column_identifier_lookup = {qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION};
|
||||
if (tryBindIdentifierToAliases(column_identifier_lookup, scope))
|
||||
break;
|
||||
|
||||
if (table_expression_data.should_qualify_columns &&
|
||||
tryBindIdentifierToTableExpressions(column_identifier_lookup, table_expression_node, scope))
|
||||
break;
|
||||
|
||||
qualified_identifier = std::move(qualified_identifier_with_removed_part);
|
||||
}
|
||||
|
||||
auto qualified_identifier_full_name = qualified_identifier.getFullName();
|
||||
node_to_projection_name.emplace(result_expression, std::move(qualified_identifier_full_name));
|
||||
|
||||
return result_expression;
|
||||
};
|
||||
|
||||
/** If identifier first part binds to some column start or table has full identifier name. Then we can try to find whole identifier in table.
|
||||
* 1. Try to bind identifier first part to column in table, if true get full identifier from table or throw exception.
|
||||
* 2. Try to bind identifier first part to table name or storage alias, if true remove first part and try to get full identifier from table or throw exception.
|
||||
@ -2988,24 +3011,35 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id
|
||||
* 3. Try to bind identifier first parts to database name and table name, if true remove first two parts and try to get full identifier from table or throw exception.
|
||||
*/
|
||||
if (table_expression_data.hasFullIdentifierName(IdentifierView(identifier)))
|
||||
return resolve_identifier_from_storage_or_throw(0 /*identifier_column_qualifier_parts*/);
|
||||
return tryResolveIdentifierFromStorage(identifier, table_expression_node, table_expression_data, scope, 0 /*identifier_column_qualifier_parts*/);
|
||||
|
||||
if (table_expression_data.canBindIdentifier(IdentifierView(identifier)))
|
||||
return resolve_identifier_from_storage_or_throw(0 /*identifier_column_qualifier_parts*/);
|
||||
{
|
||||
/** This check is insufficient to determine whether and identifier can be resolved from table expression.
|
||||
* A further check will be performed in `tryResolveIdentifierFromStorage` to see if we have such a subcolumn.
|
||||
* In cases where the subcolumn cannot be found we want to have `nullptr` instead of exception.
|
||||
* So, we set `can_be_not_found = true` to have an attempt to resolve the identifier from another table expression.
|
||||
* Example: `SELECT t.t from (SELECT 1 as t) AS a FULL JOIN (SELECT 1 as t) as t ON a.t = t.t;`
|
||||
* Initially, we will try to resolve t.t from `a` because `t.` is bound to `1 as t`. However, as it is not a nested column, we will need to resolve it from the second table expression.
|
||||
*/
|
||||
auto resolved_identifier = tryResolveIdentifierFromStorage(identifier, table_expression_node, table_expression_data, scope, 0 /*identifier_column_qualifier_parts*/, true /*can_be_not_found*/);
|
||||
if (resolved_identifier)
|
||||
return resolved_identifier;
|
||||
}
|
||||
|
||||
if (identifier.getPartsSize() == 1)
|
||||
return {};
|
||||
|
||||
const auto & table_name = table_expression_data.table_name;
|
||||
if ((!table_name.empty() && path_start == table_name) || (table_expression_node->hasAlias() && path_start == table_expression_node->getAlias()))
|
||||
return resolve_identifier_from_storage_or_throw(1 /*identifier_column_qualifier_parts*/);
|
||||
return tryResolveIdentifierFromStorage(identifier, table_expression_node, table_expression_data, scope, 1 /*identifier_column_qualifier_parts*/);
|
||||
|
||||
if (identifier.getPartsSize() == 2)
|
||||
return {};
|
||||
|
||||
const auto & database_name = table_expression_data.database_name;
|
||||
if (!database_name.empty() && path_start == database_name && identifier[1] == table_name)
|
||||
return resolve_identifier_from_storage_or_throw(2 /*identifier_column_qualifier_parts*/);
|
||||
return tryResolveIdentifierFromStorage(identifier, table_expression_node, table_expression_data, scope, 2 /*identifier_column_qualifier_parts*/);
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
@ -591,8 +591,8 @@ if (ENABLE_TESTS)
|
||||
)
|
||||
|
||||
target_link_libraries(unit_tests_dbms PRIVATE
|
||||
ch_contrib::gtest_all
|
||||
ch_contrib::gmock_all
|
||||
ch_contrib::gtest
|
||||
clickhouse_functions
|
||||
clickhouse_aggregate_functions
|
||||
clickhouse_parsers
|
||||
|
||||
@ -15,7 +15,6 @@
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/getNumberOfPhysicalCPUCores.h>
|
||||
#include <Common/tests/gtest_global_context.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/UTF8Helpers.h>
|
||||
#include <Common/TerminalSize.h>
|
||||
@ -47,6 +46,7 @@
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/Kusto/ParserKQLStatement.h>
|
||||
#include <Parsers/PRQL/ParserPRQLQuery.h>
|
||||
#include <Parsers/Kusto/parseKQLQuery.h>
|
||||
|
||||
#include <Processors/Formats/Impl/NullFormat.h>
|
||||
#include <Processors/Formats/IInputFormat.h>
|
||||
@ -349,7 +349,10 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
|
||||
if (is_interactive || ignore_error)
|
||||
{
|
||||
String message;
|
||||
res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
if (dialect == Dialect::kusto)
|
||||
res = tryParseKQLQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
else
|
||||
res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
|
||||
if (!res)
|
||||
{
|
||||
@ -359,7 +362,10 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
|
||||
}
|
||||
else
|
||||
{
|
||||
res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
if (dialect == Dialect::kusto)
|
||||
res = parseKQLQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
else
|
||||
res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
|
||||
}
|
||||
|
||||
if (is_interactive)
|
||||
@ -2013,9 +2019,6 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
|
||||
{
|
||||
bool echo_query = echo_queries;
|
||||
|
||||
/// Test tags are started with "--" so they are interpreted as comments anyway.
|
||||
/// But if the echo is enabled we have to remove the test tags from `all_queries_text`
|
||||
/// because we don't want test tags to be echoed.
|
||||
{
|
||||
/// disable logs if expects errors
|
||||
TestHint test_hint(all_queries_text);
|
||||
@ -2023,6 +2026,9 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
|
||||
processTextAsSingleQuery("SET send_logs_level = 'fatal'");
|
||||
}
|
||||
|
||||
/// Test tags are started with "--" so they are interpreted as comments anyway.
|
||||
/// But if the echo is enabled we have to remove the test tags from `all_queries_text`
|
||||
/// because we don't want test tags to be echoed.
|
||||
size_t test_tags_length = getTestTagsLength(all_queries_text);
|
||||
|
||||
/// Several queries separated by ';'.
|
||||
|
||||
@ -34,11 +34,13 @@ ConnectionPoolWithFailover::ConnectionPoolWithFailover(
|
||||
{
|
||||
const std::string & local_hostname = getFQDNOrHostName();
|
||||
|
||||
get_priority_load_balancing.hostname_differences.resize(nested_pools.size());
|
||||
get_priority_load_balancing.hostname_prefix_distance.resize(nested_pools.size());
|
||||
get_priority_load_balancing.hostname_levenshtein_distance.resize(nested_pools.size());
|
||||
for (size_t i = 0; i < nested_pools.size(); ++i)
|
||||
{
|
||||
ConnectionPool & connection_pool = dynamic_cast<ConnectionPool &>(*nested_pools[i]);
|
||||
get_priority_load_balancing.hostname_differences[i] = getHostNameDifference(local_hostname, connection_pool.getHost());
|
||||
get_priority_load_balancing.hostname_prefix_distance[i] = getHostNamePrefixDistance(local_hostname, connection_pool.getHost());
|
||||
get_priority_load_balancing.hostname_levenshtein_distance[i] = getHostNameLevenshteinDistance(local_hostname, connection_pool.getHost());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
#include <Common/AsyncTaskExecutor.h>
|
||||
#include <base/scope_guard.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -46,8 +48,10 @@ void AsyncTaskExecutor::cancel()
|
||||
{
|
||||
std::lock_guard guard(fiber_lock);
|
||||
is_cancelled = true;
|
||||
cancelBefore();
|
||||
destroyFiber();
|
||||
{
|
||||
SCOPE_EXIT({ destroyFiber(); });
|
||||
cancelBefore();
|
||||
}
|
||||
cancelAfter();
|
||||
}
|
||||
|
||||
|
||||
@ -223,7 +223,7 @@ namespace DB
|
||||
|
||||
void CaresPTRResolver::process_possible_timeout(ares_channel channel)
|
||||
{
|
||||
/* Call ares_process() unconditonally here, even if we simply timed out
|
||||
/* Call ares_process() unconditionally here, even if we simply timed out
|
||||
above, as otherwise the ares name resolve won't timeout! */
|
||||
ares_process_fd(channel, ARES_SOCKET_BAD, ARES_SOCKET_BAD);
|
||||
}
|
||||
|
||||
@ -971,6 +971,50 @@ Dwarf::Die Dwarf::getDieAtOffset(const CompilationUnit & cu, uint64_t offset) co
|
||||
return die;
|
||||
}
|
||||
|
||||
std::optional<std::pair<std::optional<Dwarf::CompilationUnit>, uint64_t>> Dwarf::getReferenceAttribute(
|
||||
const CompilationUnit & cu, const Die & die, uint64_t attr_name) const
|
||||
{
|
||||
bool found = false;
|
||||
uint64_t value;
|
||||
uint64_t form;
|
||||
forEachAttribute(cu, die, [&](const Attribute & attr)
|
||||
{
|
||||
if (attr.spec.name == attr_name)
|
||||
{
|
||||
found = true;
|
||||
value = std::get<uint64_t>(attr.attr_value);
|
||||
form = attr.spec.form;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (!found)
|
||||
return std::nullopt;
|
||||
switch (form)
|
||||
{
|
||||
case DW_FORM_ref1:
|
||||
case DW_FORM_ref2:
|
||||
case DW_FORM_ref4:
|
||||
case DW_FORM_ref8:
|
||||
case DW_FORM_ref_udata:
|
||||
return std::make_pair(std::nullopt, cu.offset + value);
|
||||
|
||||
case DW_FORM_ref_addr:
|
||||
return std::make_pair(findCompilationUnit(value), value);
|
||||
|
||||
case DW_FORM_ref_sig8:
|
||||
/// Currently we don't use this parser for types, so no need to support this.
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Type signatures are not supported (DIE at 0x{:x}, attr 0x{:x}).", die.offset, attr_name);
|
||||
|
||||
case DW_FORM_ref_sup4:
|
||||
case DW_FORM_ref_sup8:
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Supplementary object files are not supported (DIE at 0x{:x}, attr 0x{:x}).", die.offset, attr_name);
|
||||
|
||||
default:
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Unexpected form of attribute 0x{:x}: 0x{:x} (DIE at 0x{:x}).", attr_name, form, die.offset);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the @locationInfo for @address in the compilation unit represented
|
||||
* by the @sp .debug_info entry.
|
||||
@ -1300,51 +1344,24 @@ void Dwarf::findInlinedSubroutineDieForAddress(
|
||||
location.file = line_vm.getFullFileName(*call_file);
|
||||
location.line = *call_line;
|
||||
|
||||
/// Something wrong with receiving debug info about inline.
|
||||
/// If set to true we stop parsing DWARF.
|
||||
bool die_for_inline_broken = false;
|
||||
|
||||
auto get_function_name = [&](const CompilationUnit & srcu, uint64_t die_offset)
|
||||
{
|
||||
Die decl_die = getDieAtOffset(srcu, die_offset);
|
||||
auto & die_to_look_for_name = decl_die;
|
||||
Die die_to_look_for_name = getDieAtOffset(srcu, die_offset);
|
||||
|
||||
Die def_die;
|
||||
// Jump to the actual function definition instead of declaration for name
|
||||
// and line info.
|
||||
// DW_AT_specification: Incomplete, non-defining, or separate declaration
|
||||
// corresponding to a declaration
|
||||
auto offset = getAttribute<uint64_t>(srcu, decl_die, DW_AT_specification);
|
||||
if (offset)
|
||||
auto def = getReferenceAttribute(srcu, die_to_look_for_name, DW_AT_specification);
|
||||
if (def.has_value())
|
||||
{
|
||||
/// FIXME: actually it's a bug in our DWARF parser.
|
||||
///
|
||||
/// Most of the times compilation unit offset (srcu.offset) is some big number inside .debug_info (like 434782255).
|
||||
/// Offset of DIE definition is some small relative number to srcu.offset (like 3518).
|
||||
/// However in some unknown cases offset looks like global, non relative number (like 434672579) and in this
|
||||
/// case we obviously doing something wrong parsing DWARF.
|
||||
///
|
||||
/// What is important -- this bug? reproduces only with -flto=thin in release mode.
|
||||
/// Also llvm-dwarfdump --verify ./clickhouse says that our DWARF is ok, so it's another prove
|
||||
/// that we just doing something wrong.
|
||||
///
|
||||
/// FIXME: Currently we just give up parsing DWARF for inlines when we got into this situation.
|
||||
if (srcu.offset + offset.value() >= info_.size())
|
||||
{
|
||||
die_for_inline_broken = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
def_die = getDieAtOffset(srcu, srcu.offset + offset.value());
|
||||
die_to_look_for_name = def_die;
|
||||
}
|
||||
auto [def_cu, def_offset] = std::move(def.value());
|
||||
const CompilationUnit & def_cu_ref = def_cu.has_value() ? def_cu.value() : srcu;
|
||||
die_to_look_for_name = getDieAtOffset(def_cu_ref, def_offset);
|
||||
}
|
||||
|
||||
std::string_view name;
|
||||
|
||||
if (die_for_inline_broken)
|
||||
return name;
|
||||
|
||||
// The file and line will be set in the next inline subroutine based on
|
||||
// its DW_AT_call_file and DW_AT_call_line.
|
||||
forEachAttribute(srcu, die_to_look_for_name, [&](const Attribute & attr)
|
||||
@ -1386,10 +1403,6 @@ void Dwarf::findInlinedSubroutineDieForAddress(
|
||||
? get_function_name(cu, cu.offset + *abstract_origin)
|
||||
: get_function_name(findCompilationUnit(*abstract_origin), *abstract_origin);
|
||||
|
||||
/// FIXME: see comment above
|
||||
if (die_for_inline_broken)
|
||||
return false;
|
||||
|
||||
locations.push_back(location);
|
||||
|
||||
findInlinedSubroutineDieForAddress(cu, child_die, line_vm, address, base_addr_cu, locations, max_size);
|
||||
|
||||
@ -453,22 +453,11 @@ private:
|
||||
// Finds the Compilation Unit starting at offset.
|
||||
CompilationUnit findCompilationUnit(uint64_t targetOffset) const;
|
||||
|
||||
|
||||
template <class T>
|
||||
std::optional<T> getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const
|
||||
{
|
||||
std::optional<T> result;
|
||||
forEachAttribute(cu, die, [&](const Attribute & attr)
|
||||
{
|
||||
if (attr.spec.name == attr_name)
|
||||
{
|
||||
result = std::get<T>(attr.attr_value);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
// Parses an attribute of "reference" form class, i.e. a reference to another DIE.
|
||||
// Returns the unit containing the target DIE (nullopt if it's in the same unit as the source DIE)
|
||||
// and the offset of the target DIE (relative to .debug_info, not to unit).
|
||||
std::optional<std::pair<std::optional<CompilationUnit>, uint64_t>> getReferenceAttribute(
|
||||
const CompilationUnit & cu, const Die & die, uint64_t attr_name) const;
|
||||
|
||||
// Check if the given address is in the range list at the given offset in .debug_ranges.
|
||||
bool isAddrInRangeList(
|
||||
|
||||
@ -15,9 +15,14 @@ std::function<Priority(size_t index)> GetPriorityForLoadBalancing::getPriorityFu
|
||||
switch (load_balance)
|
||||
{
|
||||
case LoadBalancing::NEAREST_HOSTNAME:
|
||||
if (hostname_differences.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "It's a bug: hostname_differences is not initialized");
|
||||
get_priority = [this](size_t i) { return Priority{static_cast<Int64>(hostname_differences[i])}; };
|
||||
if (hostname_prefix_distance.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "It's a bug: hostname_prefix_distance is not initialized");
|
||||
get_priority = [this](size_t i) { return Priority{static_cast<Int64>(hostname_prefix_distance[i])}; };
|
||||
break;
|
||||
case LoadBalancing::HOSTNAME_LEVENSHTEIN_DISTANCE:
|
||||
if (hostname_levenshtein_distance.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "It's a bug: hostname_levenshtein_distance is not initialized");
|
||||
get_priority = [this](size_t i) { return Priority{static_cast<Int64>(hostname_levenshtein_distance[i])}; };
|
||||
break;
|
||||
case LoadBalancing::IN_ORDER:
|
||||
get_priority = [](size_t i) { return Priority{static_cast<Int64>(i)}; };
|
||||
|
||||
@ -13,7 +13,9 @@ public:
|
||||
|
||||
bool operator == (const GetPriorityForLoadBalancing & other) const
|
||||
{
|
||||
return load_balancing == other.load_balancing && hostname_differences == other.hostname_differences;
|
||||
return load_balancing == other.load_balancing
|
||||
&& hostname_prefix_distance == other.hostname_prefix_distance
|
||||
&& hostname_levenshtein_distance == other.hostname_levenshtein_distance;
|
||||
}
|
||||
|
||||
bool operator != (const GetPriorityForLoadBalancing & other) const
|
||||
@ -23,7 +25,8 @@ public:
|
||||
|
||||
std::function<Priority(size_t index)> getPriorityFunc(LoadBalancing load_balance, size_t offset, size_t pool_size) const;
|
||||
|
||||
std::vector<size_t> hostname_differences; /// Distances from name of this host to the names of hosts of pools.
|
||||
std::vector<size_t> hostname_prefix_distance; /// Prefix distances from name of this host to the names of hosts of pools.
|
||||
std::vector<size_t> hostname_levenshtein_distance; /// Levenshtein Distances from name of this host to the names of hosts of pools.
|
||||
|
||||
LoadBalancing load_balancing = LoadBalancing::RANDOM;
|
||||
|
||||
|
||||
@ -997,6 +997,7 @@ protected:
|
||||
--m_size;
|
||||
buf[place_value].setZero();
|
||||
inserted = false;
|
||||
keyHolderDiscardKey(key_holder);
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -1273,6 +1274,10 @@ public:
|
||||
return !buf[place_value].isZero(*this);
|
||||
}
|
||||
|
||||
bool ALWAYS_INLINE contains(const Key & x) const
|
||||
{
|
||||
return has(x);
|
||||
}
|
||||
|
||||
void write(DB::WriteBuffer & wb) const
|
||||
{
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
#include <base/types.h>
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/levenshteinDistance.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
@ -29,31 +30,6 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
static size_t levenshteinDistance(const String & lhs, const String & rhs)
|
||||
{
|
||||
size_t m = lhs.size();
|
||||
size_t n = rhs.size();
|
||||
|
||||
PODArrayWithStackMemory<size_t, 64> row(n + 1);
|
||||
|
||||
for (size_t i = 1; i <= n; ++i)
|
||||
row[i] = i;
|
||||
|
||||
for (size_t j = 1; j <= m; ++j)
|
||||
{
|
||||
row[0] = j;
|
||||
size_t prev = j - 1;
|
||||
for (size_t i = 1; i <= n; ++i)
|
||||
{
|
||||
size_t old = row[i];
|
||||
row[i] = std::min(prev + (std::tolower(lhs[j - 1]) != std::tolower(rhs[i - 1])),
|
||||
std::min(row[i - 1], row[i]) + 1);
|
||||
prev = old;
|
||||
}
|
||||
}
|
||||
return row[n];
|
||||
}
|
||||
|
||||
static void appendToQueue(size_t ind, const String & name, DistanceIndexQueue & queue, const std::vector<String> & prompting_strings)
|
||||
{
|
||||
const String & prompt = prompting_strings[ind];
|
||||
|
||||
@ -418,7 +418,7 @@ finish:
|
||||
/// this two vals are useless, xxx|xxx cannot be trivial nor prefix.
|
||||
bool next_is_trivial = true;
|
||||
pos = analyzeImpl(regexp, pos, required_substring, next_is_trivial, next_alternatives);
|
||||
/// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring.
|
||||
/// For xxx|xxx|xxx, we only combine the alternatives and return a empty required_substring.
|
||||
if (next_alternatives.empty() || shortest_literal_length(next_alternatives) < required_substring.literal.size())
|
||||
{
|
||||
global_alternatives.push_back(required_substring);
|
||||
|
||||
@ -321,7 +321,7 @@ protected:
|
||||
percolate(ptr);
|
||||
}
|
||||
|
||||
// This is equivallent to one step of bubble sort
|
||||
// This is equivalent to one step of bubble sort
|
||||
void percolate(Counter * counter)
|
||||
{
|
||||
while (counter->slot > 0)
|
||||
|
||||
@ -39,12 +39,14 @@ ZooKeeperArgs::ZooKeeperArgs(const Poco::Util::AbstractConfiguration & config, c
|
||||
throw KeeperException::fromMessage(Coordination::Error::ZBADARGUMENTS, "Timeout cannot be negative");
|
||||
|
||||
/// init get_priority_load_balancing
|
||||
get_priority_load_balancing.hostname_differences.resize(hosts.size());
|
||||
get_priority_load_balancing.hostname_prefix_distance.resize(hosts.size());
|
||||
get_priority_load_balancing.hostname_levenshtein_distance.resize(hosts.size());
|
||||
const String & local_hostname = getFQDNOrHostName();
|
||||
for (size_t i = 0; i < hosts.size(); ++i)
|
||||
{
|
||||
const String & node_host = hosts[i].substr(0, hosts[i].find_last_of(':'));
|
||||
get_priority_load_balancing.hostname_differences[i] = DB::getHostNameDifference(local_hostname, node_host);
|
||||
get_priority_load_balancing.hostname_prefix_distance[i] = DB::getHostNamePrefixDistance(local_hostname, node_host);
|
||||
get_priority_load_balancing.hostname_levenshtein_distance[i] = DB::getHostNameLevenshteinDistance(local_hostname, node_host);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <optional>
|
||||
#include <base/types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/levenshteinDistance.h>
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
|
||||
@ -121,10 +122,8 @@ bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_
|
||||
return clickhouse_port == address.port() && isLocalAddress(address.host());
|
||||
}
|
||||
|
||||
|
||||
size_t getHostNameDifference(const std::string & local_hostname, const std::string & host)
|
||||
size_t getHostNamePrefixDistance(const std::string & local_hostname, const std::string & host)
|
||||
{
|
||||
/// FIXME should we replace it with Levenstein distance? (we already have it in NamePrompter)
|
||||
size_t hostname_difference = 0;
|
||||
for (size_t i = 0; i < std::min(local_hostname.length(), host.length()); ++i)
|
||||
if (local_hostname[i] != host[i])
|
||||
@ -132,4 +131,9 @@ size_t getHostNameDifference(const std::string & local_hostname, const std::stri
|
||||
return hostname_difference;
|
||||
}
|
||||
|
||||
size_t getHostNameLevenshteinDistance(const std::string & local_hostname, const std::string & host)
|
||||
{
|
||||
return levenshteinDistance(local_hostname, host);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -26,6 +26,8 @@ namespace DB
|
||||
bool isLocalAddress(const Poco::Net::SocketAddress & address);
|
||||
bool isLocalAddress(const Poco::Net::IPAddress & address);
|
||||
|
||||
/// Returns number of different bytes in hostnames, used for load balancing
|
||||
size_t getHostNameDifference(const std::string & local_hostname, const std::string & host);
|
||||
/// Returns host name difference with name prefix, used for load balancing
|
||||
size_t getHostNamePrefixDistance(const std::string & local_hostname, const std::string & host);
|
||||
/// Returns host name difference with Levenshtein Distance.
|
||||
size_t getHostNameLevenshteinDistance(const std::string & local_hostname, const std::string & host);
|
||||
}
|
||||
|
||||
32
src/Common/levenshteinDistance.cpp
Normal file
32
src/Common/levenshteinDistance.cpp
Normal file
@ -0,0 +1,32 @@
|
||||
#include <Common/levenshteinDistance.h>
|
||||
#include <Common/PODArray.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
size_t levenshteinDistance(const String & lhs, const String & rhs)
|
||||
{
|
||||
size_t m = lhs.size();
|
||||
size_t n = rhs.size();
|
||||
|
||||
PODArrayWithStackMemory<size_t, 64> row(n + 1);
|
||||
|
||||
for (size_t i = 1; i <= n; ++i)
|
||||
row[i] = i;
|
||||
|
||||
for (size_t j = 1; j <= m; ++j)
|
||||
{
|
||||
row[0] = j;
|
||||
size_t prev = j - 1;
|
||||
for (size_t i = 1; i <= n; ++i)
|
||||
{
|
||||
size_t old = row[i];
|
||||
row[i] = std::min(prev + (std::tolower(lhs[j - 1]) != std::tolower(rhs[i - 1])),
|
||||
std::min(row[i - 1], row[i]) + 1);
|
||||
prev = old;
|
||||
}
|
||||
}
|
||||
return row[n];
|
||||
}
|
||||
|
||||
}
|
||||
12
src/Common/levenshteinDistance.h
Normal file
12
src/Common/levenshteinDistance.h
Normal file
@ -0,0 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// How many steps if we want to change lhs to rhs.
|
||||
/// Details in https://en.wikipedia.org/wiki/Levenshtein_distance
|
||||
size_t levenshteinDistance(const String & lhs, const String & rhs);
|
||||
|
||||
}
|
||||
@ -9,7 +9,6 @@ TEST(EventNotifier, SimpleTest)
|
||||
using namespace DB;
|
||||
|
||||
size_t result = 1;
|
||||
EventNotifier::init();
|
||||
|
||||
auto handler3 = EventNotifier::instance().subscribe(Coordination::Error::ZSESSIONEXPIRED, [&result](){ result *= 3; });
|
||||
|
||||
|
||||
18
src/Common/tests/gtest_main.cpp
Normal file
18
src/Common/tests/gtest_main.cpp
Normal file
@ -0,0 +1,18 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Common/tests/gtest_global_context.h>
|
||||
|
||||
class ContextEnvironment : public testing::Environment
|
||||
{
|
||||
public:
|
||||
void SetUp() override { getContext(); }
|
||||
};
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
testing::AddGlobalTestEnvironment(new ContextEnvironment);
|
||||
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/Macros.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/callOnce.h>
|
||||
|
||||
#include <Core/ServerSettings.h>
|
||||
|
||||
@ -14,6 +15,7 @@
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event ContextLock;
|
||||
extern const Event ContextLockWaitMicroseconds;
|
||||
}
|
||||
|
||||
namespace CurrentMetrics
|
||||
@ -39,8 +41,8 @@ struct ContextSharedPart : boost::noncopyable
|
||||
: macros(std::make_unique<Macros>())
|
||||
{}
|
||||
|
||||
/// For access of most of shared objects. Recursive mutex.
|
||||
mutable std::recursive_mutex mutex;
|
||||
/// For access of most of shared objects.
|
||||
mutable SharedMutex mutex;
|
||||
|
||||
mutable std::mutex keeper_dispatcher_mutex;
|
||||
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
|
||||
@ -50,13 +52,16 @@ struct ContextSharedPart : boost::noncopyable
|
||||
String path; /// Path to the data directory, with a slash at the end.
|
||||
ConfigurationPtr config; /// Global configuration settings.
|
||||
MultiVersion<Macros> macros; /// Substitutions extracted from config.
|
||||
OnceFlag schedule_pool_initialized;
|
||||
mutable std::unique_ptr<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background
|
||||
RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml
|
||||
///
|
||||
|
||||
mutable OnceFlag readers_initialized;
|
||||
mutable std::unique_ptr<IAsynchronousReader> asynchronous_remote_fs_reader;
|
||||
mutable std::unique_ptr<IAsynchronousReader> asynchronous_local_fs_reader;
|
||||
mutable std::unique_ptr<IAsynchronousReader> synchronous_local_fs_reader;
|
||||
|
||||
mutable OnceFlag threadpool_writer_initialized;
|
||||
mutable std::unique_ptr<ThreadPool> threadpool_writer;
|
||||
|
||||
mutable ThrottlerPtr remote_read_throttler; /// A server-wide throttler for remote IO reads
|
||||
@ -64,13 +69,14 @@ struct ContextSharedPart : boost::noncopyable
|
||||
|
||||
mutable ThrottlerPtr local_read_throttler; /// A server-wide throttler for local IO reads
|
||||
mutable ThrottlerPtr local_write_throttler; /// A server-wide throttler for local IO writes
|
||||
|
||||
};
|
||||
|
||||
ContextData::ContextData() = default;
|
||||
ContextData::ContextData(const ContextData &) = default;
|
||||
|
||||
Context::Context() = default;
|
||||
Context::Context(const Context & rhs) : ContextData(rhs), std::enable_shared_from_this<Context>(rhs) {}
|
||||
Context::~Context() = default;
|
||||
Context::Context(const Context &) = default;
|
||||
Context & Context::operator=(const Context &) = default;
|
||||
|
||||
SharedContextHolder::SharedContextHolder(SharedContextHolder &&) noexcept = default;
|
||||
SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) noexcept = default;
|
||||
@ -87,10 +93,10 @@ void Context::makeGlobalContext()
|
||||
global_context = shared_from_this();
|
||||
}
|
||||
|
||||
ContextMutablePtr Context::createGlobal(ContextSharedPart * shared)
|
||||
ContextMutablePtr Context::createGlobal(ContextSharedPart * shared_part)
|
||||
{
|
||||
auto res = std::shared_ptr<Context>(new Context);
|
||||
res->shared = shared;
|
||||
res->shared = shared_part;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -105,6 +111,7 @@ SharedContextHolder Context::createShared()
|
||||
return SharedContextHolder(std::make_unique<ContextSharedPart>());
|
||||
}
|
||||
|
||||
|
||||
ContextMutablePtr Context::getGlobalContext() const
|
||||
{
|
||||
auto ptr = global_context.lock();
|
||||
@ -112,22 +119,55 @@ ContextMutablePtr Context::getGlobalContext() const
|
||||
return ptr;
|
||||
}
|
||||
|
||||
std::unique_lock<std::recursive_mutex> Context::getLock() const
|
||||
std::unique_lock<SharedMutex> Context::getGlobalLock() const
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ContextLock);
|
||||
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
|
||||
return std::unique_lock(shared->mutex);
|
||||
Stopwatch watch;
|
||||
auto lock = std::unique_lock(shared->mutex);
|
||||
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
|
||||
return lock;
|
||||
}
|
||||
|
||||
std::shared_lock<SharedMutex> Context::getGlobalSharedLock() const
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ContextLock);
|
||||
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
|
||||
Stopwatch watch;
|
||||
auto lock = std::shared_lock(shared->mutex);
|
||||
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
|
||||
return lock;
|
||||
}
|
||||
|
||||
std::unique_lock<SharedMutex> Context::getLocalLock() const
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ContextLock);
|
||||
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
|
||||
Stopwatch watch;
|
||||
auto lock = std::unique_lock(mutex);
|
||||
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
|
||||
return lock;
|
||||
}
|
||||
|
||||
std::shared_lock<SharedMutex> Context::getLocalSharedLock() const
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ContextLock);
|
||||
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
|
||||
Stopwatch watch;
|
||||
auto lock = std::shared_lock(mutex);
|
||||
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
|
||||
return lock;
|
||||
}
|
||||
|
||||
String Context::getPath() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
auto lock = getGlobalSharedLock();
|
||||
return shared->path;
|
||||
}
|
||||
|
||||
void Context::setPath(const String & path)
|
||||
{
|
||||
auto lock = getLock();
|
||||
auto lock = getGlobalLock();
|
||||
shared->path = path;
|
||||
}
|
||||
|
||||
@ -143,15 +183,13 @@ void Context::setMacros(std::unique_ptr<Macros> && macros)
|
||||
|
||||
BackgroundSchedulePool & Context::getSchedulePool() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
if (!shared->schedule_pool)
|
||||
{
|
||||
callOnce(shared->schedule_pool_initialized, [&] {
|
||||
shared->schedule_pool = std::make_unique<BackgroundSchedulePool>(
|
||||
shared->server_settings.background_schedule_pool_size,
|
||||
CurrentMetrics::BackgroundSchedulePoolTask,
|
||||
CurrentMetrics::BackgroundSchedulePoolSize,
|
||||
"BgSchPool");
|
||||
}
|
||||
});
|
||||
|
||||
return *shared->schedule_pool;
|
||||
}
|
||||
@ -168,30 +206,21 @@ const RemoteHostFilter & Context::getRemoteHostFilter() const
|
||||
|
||||
IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) const
|
||||
{
|
||||
auto lock = getLock();
|
||||
callOnce(shared->readers_initialized, [&] {
|
||||
const auto & config = getConfigRef();
|
||||
shared->asynchronous_remote_fs_reader = createThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER, config);
|
||||
shared->asynchronous_local_fs_reader = createThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER, config);
|
||||
shared->synchronous_local_fs_reader = createThreadPoolReader(FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER, config);
|
||||
});
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
|
||||
{
|
||||
if (!shared->asynchronous_remote_fs_reader)
|
||||
shared->asynchronous_remote_fs_reader = createThreadPoolReader(type, getConfigRef());
|
||||
return *shared->asynchronous_remote_fs_reader;
|
||||
}
|
||||
case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
|
||||
{
|
||||
if (!shared->asynchronous_local_fs_reader)
|
||||
shared->asynchronous_local_fs_reader = createThreadPoolReader(type, getConfigRef());
|
||||
|
||||
return *shared->asynchronous_local_fs_reader;
|
||||
}
|
||||
case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
|
||||
{
|
||||
if (!shared->synchronous_local_fs_reader)
|
||||
shared->synchronous_local_fs_reader = createThreadPoolReader(type, getConfigRef());
|
||||
|
||||
return *shared->synchronous_local_fs_reader;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -207,19 +236,19 @@ std::shared_ptr<FilesystemReadPrefetchesLog> Context::getFilesystemReadPrefetche
|
||||
|
||||
void Context::setConfig(const ConfigurationPtr & config)
|
||||
{
|
||||
auto lock = getLock();
|
||||
auto lock = getGlobalLock();
|
||||
shared->config = config;
|
||||
}
|
||||
|
||||
const Poco::Util::AbstractConfiguration & Context::getConfigRef() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
auto lock = getGlobalSharedLock();
|
||||
return shared->config ? *shared->config : Poco::Util::Application::instance().config();
|
||||
}
|
||||
|
||||
std::shared_ptr<AsyncReadCounters> Context::getAsyncReadCounters() const
|
||||
{
|
||||
auto lock = getLock();
|
||||
auto lock = getLocalLock();
|
||||
if (!async_read_counters)
|
||||
async_read_counters = std::make_shared<AsyncReadCounters>();
|
||||
return async_read_counters;
|
||||
@ -227,18 +256,14 @@ std::shared_ptr<AsyncReadCounters> Context::getAsyncReadCounters() const
|
||||
|
||||
ThreadPool & Context::getThreadPoolWriter() const
|
||||
{
|
||||
const auto & config = getConfigRef();
|
||||
|
||||
auto lock = getLock();
|
||||
|
||||
if (!shared->threadpool_writer)
|
||||
{
|
||||
callOnce(shared->threadpool_writer_initialized, [&] {
|
||||
const auto & config = getConfigRef();
|
||||
auto pool_size = config.getUInt(".threadpool_writer_pool_size", 100);
|
||||
auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000);
|
||||
|
||||
shared->threadpool_writer = std::make_unique<ThreadPool>(
|
||||
CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, pool_size, pool_size, queue_size);
|
||||
}
|
||||
});
|
||||
|
||||
return *shared->threadpool_writer;
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
#include <Common/MultiVersion.h>
|
||||
#include <Common/RemoteHostFilter.h>
|
||||
#include <Common/SharedMutex.h>
|
||||
|
||||
#include <Disks/IO/getThreadPoolReader.h>
|
||||
|
||||
@ -44,17 +45,9 @@ private:
|
||||
std::unique_ptr<ContextSharedPart> shared;
|
||||
};
|
||||
|
||||
|
||||
class Context : public std::enable_shared_from_this<Context>
|
||||
class ContextData
|
||||
{
|
||||
private:
|
||||
/// Use copy constructor or createGlobal() instead
|
||||
Context();
|
||||
Context(const Context &);
|
||||
Context & operator=(const Context &);
|
||||
|
||||
std::unique_lock<std::recursive_mutex> getLock() const;
|
||||
|
||||
protected:
|
||||
ContextWeakMutablePtr global_context;
|
||||
inline static ContextPtr global_context_instance;
|
||||
ContextSharedPart * shared;
|
||||
@ -63,9 +56,33 @@ private:
|
||||
mutable std::shared_ptr<AsyncReadCounters> async_read_counters;
|
||||
|
||||
Settings settings; /// Setting for query execution.
|
||||
|
||||
public:
|
||||
/// Use copy constructor or createGlobal() instead
|
||||
ContextData();
|
||||
ContextData(const ContextData &);
|
||||
};
|
||||
|
||||
class Context : public ContextData, public std::enable_shared_from_this<Context>
|
||||
{
|
||||
private:
|
||||
/// ContextData mutex
|
||||
mutable SharedMutex mutex;
|
||||
|
||||
Context();
|
||||
Context(const Context &);
|
||||
|
||||
std::unique_lock<SharedMutex> getGlobalLock() const;
|
||||
|
||||
std::shared_lock<SharedMutex> getGlobalSharedLock() const;
|
||||
|
||||
std::unique_lock<SharedMutex> getLocalLock() const;
|
||||
|
||||
std::shared_lock<SharedMutex> getLocalSharedLock() const;
|
||||
|
||||
public:
|
||||
/// Create initial Context with ContextShared and etc.
|
||||
static ContextMutablePtr createGlobal(ContextSharedPart * shared);
|
||||
static ContextMutablePtr createGlobal(ContextSharedPart * shared_part);
|
||||
static SharedContextHolder createShared();
|
||||
|
||||
ContextMutablePtr getGlobalContext() const;
|
||||
|
||||
@ -284,7 +284,7 @@ void deserializeLogMagic(ReadBuffer & in)
|
||||
/// strange, that this 550 bytes obviously was a part of Create transaction,
|
||||
/// but the operation code was -1. We have added debug prints to original
|
||||
/// zookeeper (3.6.3) and found that it just reads 550 bytes of this "Error"
|
||||
/// transaction, tooks the first 4 bytes as an error code (it was 79, non
|
||||
/// transaction, took the first 4 bytes as an error code (it was 79, non
|
||||
/// existing code) and skip all remaining 546 bytes. NOTE: it looks like a bug
|
||||
/// in ZooKeeper.
|
||||
///
|
||||
|
||||
@ -71,6 +71,13 @@ protected:
|
||||
DB::KeeperContextPtr keeper_context = std::make_shared<DB::KeeperContext>(true);
|
||||
Poco::Logger * log{&Poco::Logger::get("CoordinationTest")};
|
||||
|
||||
void SetUp() override
|
||||
{
|
||||
Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
|
||||
Poco::Logger::root().setChannel(channel);
|
||||
Poco::Logger::root().setLevel("trace");
|
||||
}
|
||||
|
||||
void setLogDirectory(const std::string & path) { keeper_context->setLogDisk(std::make_shared<DB::DiskLocal>("LogDisk", path)); }
|
||||
|
||||
void setSnapshotDirectory(const std::string & path)
|
||||
@ -2911,13 +2918,4 @@ INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite,
|
||||
CoordinationTest,
|
||||
::testing::ValuesIn(std::initializer_list<CompressionParam>{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}}));
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
|
||||
Poco::Logger::root().setChannel(channel);
|
||||
Poco::Logger::root().setLevel("trace");
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
65
src/Core/MySQL/MySQLUtils.cpp
Normal file
65
src/Core/MySQL/MySQLUtils.cpp
Normal file
@ -0,0 +1,65 @@
|
||||
#include "Common/assert_cast.h"
|
||||
#include "Columns/ColumnNullable.h"
|
||||
#include "Columns/ColumnsDateTime.h"
|
||||
#include "Core/DecimalFunctions.h"
|
||||
#include "DataTypes/IDataType.h"
|
||||
#include "base/types.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace MySQLProtocol
|
||||
{
|
||||
namespace MySQLUtils
|
||||
{
|
||||
|
||||
DecimalUtils::DecimalComponents<DateTime64>
|
||||
getNormalizedDateTime64Components(DataTypePtr data_type, ColumnPtr col, size_t row_num)
|
||||
{
|
||||
const auto * date_time_type = typeid_cast<const DataTypeDateTime64 *>(data_type.get());
|
||||
|
||||
static constexpr UInt32 MaxScale = DecimalUtils::max_precision<DateTime64>;
|
||||
UInt32 scale = std::min(MaxScale, date_time_type->getScale());
|
||||
|
||||
const auto value = assert_cast<const ColumnDateTime64 &>(*col).getData()[row_num];
|
||||
auto components = DecimalUtils::split(value, scale);
|
||||
|
||||
using T = typename DateTime64::NativeType;
|
||||
if (value.value < 0 && components.fractional)
|
||||
{
|
||||
components.fractional = DecimalUtils::scaleMultiplier<T>(scale) + (components.whole ? T(-1) : T(1)) * components.fractional;
|
||||
--components.whole;
|
||||
}
|
||||
|
||||
if (components.fractional != 0)
|
||||
{
|
||||
if (scale > 6)
|
||||
{
|
||||
// MySQL Timestamp has max scale of 6
|
||||
components.fractional /= static_cast<int>(pow(10, scale - 6));
|
||||
}
|
||||
else
|
||||
{
|
||||
// fractional == 1 is a different microsecond value depending on the scale
|
||||
// Scale 1 = 100000
|
||||
// Scale 2 = 010000
|
||||
// Scale 3 = 001000
|
||||
// Scale 4 = 000100
|
||||
// Scale 5 = 000010
|
||||
// Scale 6 = 000001
|
||||
components.fractional *= static_cast<int>(pow(10, 6 - scale));
|
||||
}
|
||||
}
|
||||
|
||||
return components;
|
||||
};
|
||||
|
||||
ColumnPtr getBaseColumn(const DB::Columns & columns, size_t i)
|
||||
{
|
||||
ColumnPtr col = columns[i]->convertToFullIfNeeded();
|
||||
if (col->isNullable())
|
||||
return assert_cast<const ColumnNullable &>(*col).getNestedColumnPtr();
|
||||
return col;
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
22
src/Core/MySQL/MySQLUtils.h
Normal file
22
src/Core/MySQL/MySQLUtils.h
Normal file
@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include "Core/DecimalFunctions.h"
|
||||
#include "DataTypes/IDataType.h"
|
||||
#include "base/types.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace MySQLProtocol
|
||||
{
|
||||
namespace MySQLUtils
|
||||
{
|
||||
/// Splits DateTime64 column data at a certain row number into whole and fractional part
|
||||
/// Additionally, normalizes the fractional part as if it was scale 6 for MySQL compatibility purposes
|
||||
DecimalUtils::DecimalComponents<DateTime64> getNormalizedDateTime64Components(DataTypePtr data_type, ColumnPtr col, size_t row_num);
|
||||
|
||||
/// If a column is ColumnSparse/ColumnLowCardinality/ColumnNullable, it is unwrapped in a correct order;
|
||||
/// otherwise, the original column is returned
|
||||
ColumnPtr getBaseColumn(const DB::Columns & columns, size_t i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -15,6 +15,7 @@
|
||||
#include "DataTypes/DataTypesNumber.h"
|
||||
#include "Formats/FormatSettings.h"
|
||||
#include "IO/WriteBufferFromString.h"
|
||||
#include "MySQLUtils.h"
|
||||
#include "base/DayNum.h"
|
||||
#include "base/Decimal.h"
|
||||
#include "base/types.h"
|
||||
@ -25,14 +26,14 @@ namespace MySQLProtocol
|
||||
{
|
||||
namespace ProtocolBinary
|
||||
{
|
||||
ResultSetRow::ResultSetRow(const Serializations & serializations_, const DataTypes & data_types_, const Columns & columns_, int row_num_)
|
||||
ResultSetRow::ResultSetRow(const Serializations & serializations_, const DataTypes & data_types_, const Columns & columns_, size_t row_num_)
|
||||
: row_num(row_num_), columns(columns_), data_types(data_types_), serializations(serializations_)
|
||||
{
|
||||
payload_size = 1 + null_bitmap_size;
|
||||
FormatSettings format_settings;
|
||||
for (size_t i = 0; i < columns.size(); ++i)
|
||||
{
|
||||
ColumnPtr col = getColumn(i);
|
||||
ColumnPtr col = MySQLUtils::getBaseColumn(columns, i);
|
||||
if (col->isNullAt(row_num))
|
||||
{
|
||||
// See https://dev.mysql.com/doc/dev/mysql-server/8.1.0/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row
|
||||
@ -42,7 +43,7 @@ ResultSetRow::ResultSetRow(const Serializations & serializations_, const DataTyp
|
||||
continue; // NULLs are stored in the null bitmap only
|
||||
}
|
||||
|
||||
DataTypePtr data_type = removeLowCardinality(removeNullable((data_types[i])));
|
||||
DataTypePtr data_type = removeLowCardinalityAndNullable(data_types[i]);
|
||||
TypeIndex type_index = data_type->getTypeId();
|
||||
switch (type_index)
|
||||
{
|
||||
@ -95,13 +96,7 @@ ResultSetRow::ResultSetRow(const Serializations & serializations_, const DataTyp
|
||||
break;
|
||||
}
|
||||
case TypeIndex::DateTime64: {
|
||||
auto [components, scale] = getDateTime64ComponentsWithScale(data_type, col);
|
||||
if (scale > 6)
|
||||
{
|
||||
// MySQL Timestamp has max scale of 6
|
||||
components.fractional /= static_cast<int>(pow(10, scale - 6));
|
||||
}
|
||||
|
||||
auto components = MySQLUtils::getNormalizedDateTime64Components(data_type, col, row_num);
|
||||
LocalDateTime ldt = LocalDateTime(components.whole, DateLUT::instance(getDateTimeTimezone(*data_type)));
|
||||
|
||||
bool has_microseconds = components.fractional != 0;
|
||||
@ -150,11 +145,11 @@ void ResultSetRow::writePayloadImpl(WriteBuffer & buffer) const
|
||||
buffer.write(null_bitmap.data(), null_bitmap_size);
|
||||
for (size_t i = 0; i < columns.size(); ++i)
|
||||
{
|
||||
ColumnPtr col = getColumn(i);
|
||||
ColumnPtr col = MySQLUtils::getBaseColumn(columns, i);
|
||||
if (col->isNullAt(row_num))
|
||||
continue;
|
||||
|
||||
DataTypePtr data_type = removeLowCardinality(removeNullable((data_types[i])));
|
||||
DataTypePtr data_type = removeLowCardinalityAndNullable(data_types[i]);
|
||||
TypeIndex type_index = data_type->getTypeId();
|
||||
switch (type_index)
|
||||
{
|
||||
@ -257,27 +252,7 @@ void ResultSetRow::writePayloadImpl(WriteBuffer & buffer) const
|
||||
break;
|
||||
}
|
||||
case TypeIndex::DateTime64: {
|
||||
auto [components, scale] = getDateTime64ComponentsWithScale(data_type, col);
|
||||
if (components.fractional != 0)
|
||||
{
|
||||
if (scale > 6)
|
||||
{
|
||||
// MySQL Timestamp has max scale of 6
|
||||
components.fractional /= static_cast<int>(pow(10, scale - 6));
|
||||
}
|
||||
else
|
||||
{
|
||||
// fractional == 1 is a different microsecond value depending on the scale
|
||||
// Scale 1 = 100000
|
||||
// Scale 2 = 010000
|
||||
// Scale 3 = 001000
|
||||
// Scale 4 = 000100
|
||||
// Scale 5 = 000010
|
||||
// Scale 6 = 000001
|
||||
components.fractional *= static_cast<int>(pow(10, 6 - scale));
|
||||
}
|
||||
}
|
||||
|
||||
auto components = MySQLUtils::getNormalizedDateTime64Components(data_type, col, row_num);
|
||||
String timezone = getDateTimeTimezone(*data_type);
|
||||
LocalDateTime ldt = LocalDateTime(components.whole, DateLUT::instance(timezone));
|
||||
UInt16 year = ldt.year();
|
||||
@ -327,34 +302,6 @@ void ResultSetRow::writePayloadImpl(WriteBuffer & buffer) const
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ResultSetRow::DateTime64ComponentsWithScale ResultSetRow::getDateTime64ComponentsWithScale(DataTypePtr data_type, ColumnPtr col) const
|
||||
{
|
||||
const auto * date_time_type = typeid_cast<const DataTypeDateTime64 *>(data_type.get());
|
||||
|
||||
static constexpr UInt32 MaxScale = DecimalUtils::max_precision<DateTime64>;
|
||||
UInt32 scale = std::min(MaxScale, date_time_type->getScale());
|
||||
|
||||
const auto value = assert_cast<const ColumnDateTime64 &>(*col).getData()[row_num];
|
||||
auto components = DecimalUtils::split(value, scale);
|
||||
|
||||
using T = typename DateTime64::NativeType;
|
||||
if (value.value < 0 && components.fractional)
|
||||
{
|
||||
components.fractional = DecimalUtils::scaleMultiplier<T>(scale) + (components.whole ? T(-1) : T(1)) * components.fractional;
|
||||
--components.whole;
|
||||
}
|
||||
|
||||
return {components, scale};
|
||||
}
|
||||
|
||||
ColumnPtr ResultSetRow::getColumn(size_t i) const
|
||||
{
|
||||
ColumnPtr col = columns[i]->convertToFullIfNeeded();
|
||||
if (col->isNullable())
|
||||
return assert_cast<const ColumnNullable &>(*col).getNestedColumnPtr();
|
||||
return col;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,14 +16,8 @@ namespace ProtocolBinary
|
||||
{
|
||||
class ResultSetRow : public IMySQLWritePacket
|
||||
{
|
||||
using DateTime64ComponentsWithScale = std::pair<DecimalUtils::DecimalComponents<DateTime64>, UInt32>;
|
||||
|
||||
private:
|
||||
DateTime64ComponentsWithScale getDateTime64ComponentsWithScale(DataTypePtr data_type, ColumnPtr col) const;
|
||||
ColumnPtr getColumn(size_t i) const;
|
||||
|
||||
protected:
|
||||
int row_num;
|
||||
size_t row_num;
|
||||
const Columns & columns;
|
||||
const DataTypes & data_types;
|
||||
const Serializations & serializations;
|
||||
@ -41,7 +35,7 @@ protected:
|
||||
void writePayloadImpl(WriteBuffer & buffer) const override;
|
||||
|
||||
public:
|
||||
ResultSetRow(const Serializations & serializations_, const DataTypes & data_types_, const Columns & columns_, int row_num_);
|
||||
ResultSetRow(const Serializations & serializations_, const DataTypes & data_types_, const Columns & columns_, size_t row_num_);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include "DataTypes/DataTypeLowCardinality.h"
|
||||
#include "DataTypes/DataTypeNullable.h"
|
||||
#include "DataTypes/DataTypesDecimal.h"
|
||||
#include "MySQLUtils.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -17,17 +18,32 @@ namespace MySQLProtocol
|
||||
namespace ProtocolText
|
||||
{
|
||||
|
||||
ResultSetRow::ResultSetRow(const Serializations & serializations, const Columns & columns_, int row_num_)
|
||||
ResultSetRow::ResultSetRow(const Serializations & serializations, const DataTypes & data_types, const Columns & columns_, size_t row_num_)
|
||||
: columns(columns_), row_num(row_num_)
|
||||
{
|
||||
static FormatSettings format_settings = {.bool_true_representation = "1", .bool_false_representation = "0"};
|
||||
|
||||
for (size_t i = 0; i < columns.size(); ++i)
|
||||
{
|
||||
DataTypePtr data_type = removeLowCardinalityAndNullable(data_types[i]);
|
||||
TypeIndex type_index = data_type->getTypeId();
|
||||
if (columns[i]->isNullAt(row_num))
|
||||
{
|
||||
payload_size += 1;
|
||||
serialized.emplace_back("\xfb");
|
||||
}
|
||||
// Arbitrary precision DateTime64 needs to be forced into precision 6, as it is the maximum that MySQL supports
|
||||
else if (type_index == TypeIndex::DateTime64)
|
||||
{
|
||||
WriteBufferFromOwnString ostr;
|
||||
ColumnPtr col = MySQLUtils::getBaseColumn(columns, i);
|
||||
auto components = MySQLUtils::getNormalizedDateTime64Components(data_type, col, row_num);
|
||||
writeDateTimeText<'-', ':', ' '>(LocalDateTime(components.whole, DateLUT::instance(getDateTimeTimezone(*data_type))), ostr);
|
||||
ostr.write('.');
|
||||
writeDateTime64FractionalText<DateTime64>(components.fractional, 6, ostr);
|
||||
payload_size += getLengthEncodedStringSize(ostr.str());
|
||||
serialized.push_back(std::move(ostr.str()));
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteBufferFromOwnString ostr;
|
||||
@ -141,7 +157,7 @@ ColumnDefinition getColumnDefinition(const String & column_name, const DataTypeP
|
||||
CharacterSet charset = CharacterSet::binary;
|
||||
int flags = 0;
|
||||
uint8_t decimals = 0;
|
||||
DataTypePtr normalized_data_type = removeLowCardinality(removeNullable(data_type));
|
||||
DataTypePtr normalized_data_type = removeLowCardinalityAndNullable(data_type);
|
||||
TypeIndex type_index = normalized_data_type->getTypeId();
|
||||
switch (type_index)
|
||||
{
|
||||
|
||||
@ -67,7 +67,7 @@ class ResultSetRow : public IMySQLWritePacket
|
||||
{
|
||||
protected:
|
||||
const Columns & columns;
|
||||
int row_num;
|
||||
size_t row_num;
|
||||
size_t payload_size = 0;
|
||||
std::vector<String> serialized;
|
||||
|
||||
@ -76,7 +76,7 @@ protected:
|
||||
void writePayloadImpl(WriteBuffer & buffer) const override;
|
||||
|
||||
public:
|
||||
ResultSetRow(const Serializations & serializations, const Columns & columns_, int row_num_);
|
||||
ResultSetRow(const Serializations & serializations, const DataTypes & data_types, const Columns & columns_, size_t row_num_);
|
||||
};
|
||||
|
||||
class ComFieldList : public LimitedReadPacket
|
||||
|
||||
176
src/Core/Range.cpp
Normal file
176
src/Core/Range.cpp
Normal file
@ -0,0 +1,176 @@
|
||||
#include <Core/Range.h>
|
||||
#include <Common/FieldVisitorToString.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
Range::Range(const FieldRef & point) /// NOLINT
|
||||
: left(point), right(point), left_included(true), right_included(true) {}
|
||||
|
||||
/// A bounded two-sided range.
|
||||
Range::Range(const FieldRef & left_, bool left_included_, const FieldRef & right_, bool right_included_)
|
||||
: left(left_)
|
||||
, right(right_)
|
||||
, left_included(left_included_)
|
||||
, right_included(right_included_)
|
||||
{
|
||||
shrinkToIncludedIfPossible();
|
||||
}
|
||||
|
||||
Range Range::createWholeUniverse()
|
||||
{
|
||||
return Range(NEGATIVE_INFINITY, true, POSITIVE_INFINITY, true);
|
||||
}
|
||||
|
||||
Range Range::createWholeUniverseWithoutNull()
|
||||
{
|
||||
return Range(NEGATIVE_INFINITY, false, POSITIVE_INFINITY, false);
|
||||
}
|
||||
|
||||
Range Range::createRightBounded(const FieldRef & right_point, bool right_included, bool with_null)
|
||||
{
|
||||
Range r = with_null ? createWholeUniverse() : createWholeUniverseWithoutNull();
|
||||
r.right = right_point;
|
||||
r.right_included = right_included;
|
||||
r.shrinkToIncludedIfPossible();
|
||||
// Special case for [-Inf, -Inf]
|
||||
if (r.right.isNegativeInfinity() && right_included)
|
||||
r.left_included = true;
|
||||
return r;
|
||||
}
|
||||
|
||||
Range Range::createLeftBounded(const FieldRef & left_point, bool left_included, bool with_null)
|
||||
{
|
||||
Range r = with_null ? createWholeUniverse() : createWholeUniverseWithoutNull();
|
||||
r.left = left_point;
|
||||
r.left_included = left_included;
|
||||
r.shrinkToIncludedIfPossible();
|
||||
// Special case for [+Inf, +Inf]
|
||||
if (r.left.isPositiveInfinity() && left_included)
|
||||
r.right_included = true;
|
||||
return r;
|
||||
}
|
||||
|
||||
/** Optimize the range. If it has an open boundary and the Field type is "loose"
|
||||
* - then convert it to closed, narrowing by one.
|
||||
* That is, for example, turn (0,2) into [1].
|
||||
*/
|
||||
void Range::shrinkToIncludedIfPossible()
|
||||
{
|
||||
if (left.isExplicit() && !left_included)
|
||||
{
|
||||
if (left.getType() == Field::Types::UInt64 && left.get<UInt64>() != std::numeric_limits<UInt64>::max())
|
||||
{
|
||||
++left.get<UInt64 &>();
|
||||
left_included = true;
|
||||
}
|
||||
if (left.getType() == Field::Types::Int64 && left.get<Int64>() != std::numeric_limits<Int64>::max())
|
||||
{
|
||||
++left.get<Int64 &>();
|
||||
left_included = true;
|
||||
}
|
||||
}
|
||||
if (right.isExplicit() && !right_included)
|
||||
{
|
||||
if (right.getType() == Field::Types::UInt64 && right.get<UInt64>() != std::numeric_limits<UInt64>::min())
|
||||
{
|
||||
--right.get<UInt64 &>();
|
||||
right_included = true;
|
||||
}
|
||||
if (right.getType() == Field::Types::Int64 && right.get<Int64>() != std::numeric_limits<Int64>::min())
|
||||
{
|
||||
--right.get<Int64 &>();
|
||||
right_included = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
inline bool equals(const Field & lhs, const Field & rhs)
|
||||
{
|
||||
return applyVisitor(FieldVisitorAccurateEquals(), lhs, rhs);
|
||||
}
|
||||
|
||||
inline bool less(const Field & lhs, const Field & rhs)
|
||||
{
|
||||
return applyVisitor(FieldVisitorAccurateLess(), lhs, rhs);
|
||||
}
|
||||
}
|
||||
|
||||
bool Range::empty() const
|
||||
{
|
||||
return less(right, left)
|
||||
|| ((!left_included || !right_included)
|
||||
&& !less(left, right));
|
||||
}
|
||||
|
||||
/// x contained in the range
|
||||
bool Range::contains(const FieldRef & x) const
|
||||
{
|
||||
return !leftThan(x) && !rightThan(x);
|
||||
}
|
||||
|
||||
/// x is to the left
|
||||
bool Range::rightThan(const FieldRef & x) const
|
||||
{
|
||||
return less(left, x) || (left_included && equals(x, left));
|
||||
}
|
||||
|
||||
/// x is to the right
|
||||
bool Range::leftThan(const FieldRef & x) const
|
||||
{
|
||||
return less(x, right) || (right_included && equals(x, right));
|
||||
}
|
||||
|
||||
bool Range::intersectsRange(const Range & r) const
|
||||
{
|
||||
/// r to the left of me.
|
||||
if (less(r.right, left) || ((!left_included || !r.right_included) && equals(r.right, left)))
|
||||
return false;
|
||||
|
||||
/// r to the right of me.
|
||||
if (less(right, r.left) || ((!right_included || !r.left_included) && equals(r.left, right)))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Range::containsRange(const Range & r) const
|
||||
{
|
||||
/// r starts to the left of me.
|
||||
if (less(r.left, left) || (r.left_included && !left_included && equals(r.left, left)))
|
||||
return false;
|
||||
|
||||
/// r ends right of me.
|
||||
if (less(right, r.right) || (r.right_included && !right_included && equals(r.right, right)))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Range::invert()
|
||||
{
|
||||
std::swap(left, right);
|
||||
if (left.isPositiveInfinity())
|
||||
left = NEGATIVE_INFINITY;
|
||||
if (right.isNegativeInfinity())
|
||||
right = POSITIVE_INFINITY;
|
||||
std::swap(left_included, right_included);
|
||||
}
|
||||
|
||||
String Range::toString() const
|
||||
{
|
||||
WriteBufferFromOwnString str;
|
||||
|
||||
str << (left_included ? '[' : '(') << applyVisitor(FieldVisitorToString(), left) << ", ";
|
||||
str << applyVisitor(FieldVisitorToString(), right) << (right_included ? ']' : ')');
|
||||
|
||||
return str.str();
|
||||
}
|
||||
|
||||
}
|
||||
92
src/Core/Range.h
Normal file
92
src/Core/Range.h
Normal file
@ -0,0 +1,92 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Field.h>
|
||||
#include <Core/ColumnsWithTypeAndName.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
|
||||
/** Range between fields, used for index analysis
|
||||
* (various arithmetic on intervals of various forms).
|
||||
*/
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** A field, that can be stored in two representations:
|
||||
* - A standalone field.
|
||||
* - A field with reference to its position in a block.
|
||||
* It's needed for execution of functions on ranges during
|
||||
* index analysis. If function was executed once for field,
|
||||
* its result would be cached for whole block for which field's reference points to.
|
||||
*/
|
||||
struct FieldRef : public Field
|
||||
{
|
||||
FieldRef() = default;
|
||||
|
||||
/// Create as explicit field without block.
|
||||
template <typename T>
|
||||
FieldRef(T && value) : Field(std::forward<T>(value)) {} /// NOLINT
|
||||
|
||||
/// Create as reference to field in block.
|
||||
FieldRef(ColumnsWithTypeAndName * columns_, size_t row_idx_, size_t column_idx_)
|
||||
: Field((*(*columns_)[column_idx_].column)[row_idx_]),
|
||||
columns(columns_), row_idx(row_idx_), column_idx(column_idx_) {}
|
||||
|
||||
bool isExplicit() const { return columns == nullptr; }
|
||||
|
||||
ColumnsWithTypeAndName * columns = nullptr;
|
||||
size_t row_idx = 0;
|
||||
size_t column_idx = 0;
|
||||
};
|
||||
|
||||
/** Range with open or closed ends; possibly unbounded.
|
||||
*/
|
||||
struct Range
|
||||
{
|
||||
public:
|
||||
FieldRef left; /// the left border
|
||||
FieldRef right; /// the right border
|
||||
bool left_included; /// includes the left border
|
||||
bool right_included; /// includes the right border
|
||||
|
||||
/// One point.
|
||||
Range(const FieldRef & point); /// NOLINT
|
||||
|
||||
/// A bounded two-sided range.
|
||||
Range(const FieldRef & left_, bool left_included_, const FieldRef & right_, bool right_included_);
|
||||
|
||||
static Range createWholeUniverse();
|
||||
static Range createWholeUniverseWithoutNull();
|
||||
static Range createRightBounded(const FieldRef & right_point, bool right_included, bool with_null = false);
|
||||
static Range createLeftBounded(const FieldRef & left_point, bool left_included, bool with_null = false);
|
||||
|
||||
/** Optimize the range. If it has an open boundary and the Field type is "loose"
|
||||
* - then convert it to closed, narrowing by one.
|
||||
* That is, for example, turn (0,2) into [1].
|
||||
*/
|
||||
void shrinkToIncludedIfPossible();
|
||||
|
||||
bool empty() const;
|
||||
|
||||
/// x contained in the range
|
||||
bool contains(const FieldRef & x) const;
|
||||
|
||||
/// x is to the left
|
||||
bool rightThan(const FieldRef & x) const;
|
||||
|
||||
/// x is to the right
|
||||
bool leftThan(const FieldRef & x) const;
|
||||
|
||||
bool intersectsRange(const Range & r) const;
|
||||
|
||||
bool containsRange(const Range & r) const;
|
||||
|
||||
void invert();
|
||||
|
||||
String toString() const;
|
||||
};
|
||||
|
||||
/** Hyperrectangle is a product of ranges: each range across each coordinate.
|
||||
*/
|
||||
using Hyperrectangle = std::vector<Range>;
|
||||
|
||||
}
|
||||
@ -627,7 +627,7 @@ class IColumn;
|
||||
\
|
||||
M(Bool, optimize_rewrite_sum_if_to_count_if, false, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \
|
||||
M(Bool, optimize_rewrite_aggregate_function_with_if, true, "Rewrite aggregate functions with if expression as argument when logically equivalent. For example, avg(if(cond, col, null)) can be rewritten to avgIf(cond, col)", 0) \
|
||||
M(Bool, optimize_rewrite_array_exists_to_has, true, "Rewrite arrayExists() functions to has() when logically equivalent. For example, arrayExists(x -> x = 1, arr) can be rewritten to has(arr, 1)", 0) \
|
||||
M(Bool, optimize_rewrite_array_exists_to_has, false, "Rewrite arrayExists() functions to has() when logically equivalent. For example, arrayExists(x -> x = 1, arr) can be rewritten to has(arr, 1)", 0) \
|
||||
M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \
|
||||
\
|
||||
M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \
|
||||
@ -963,6 +963,7 @@ class IColumn;
|
||||
M(Bool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
|
||||
M(Bool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
|
||||
M(Bool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
|
||||
M(Bool, input_format_values_allow_data_after_semicolon, false, "For Values format: allow extra data after semicolon (used by client to interpret comments).", 0) \
|
||||
M(Bool, input_format_avro_allow_missing_fields, false, "For Avro/AvroConfluent format: when field is not found in schema use default value instead of error", 0) \
|
||||
/** This setting is obsolete and do nothing, left for compatibility reasons. */ \
|
||||
M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \
|
||||
|
||||
@ -20,6 +20,7 @@ namespace ErrorCodes
|
||||
IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING,
|
||||
{{"random", LoadBalancing::RANDOM},
|
||||
{"nearest_hostname", LoadBalancing::NEAREST_HOSTNAME},
|
||||
{"hostname_levenshtein_distance", LoadBalancing::HOSTNAME_LEVENSHTEIN_DISTANCE},
|
||||
{"in_order", LoadBalancing::IN_ORDER},
|
||||
{"first_or_random", LoadBalancing::FIRST_OR_RANDOM},
|
||||
{"round_robin", LoadBalancing::ROUND_ROBIN}})
|
||||
@ -140,7 +141,7 @@ IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS,
|
||||
{{"clickhouse", Dialect::clickhouse},
|
||||
{"kusto", Dialect::kusto},
|
||||
{"prql", Dialect::prql}})
|
||||
// FIXME: do not add 'kusto_auto' to the list. Maybe remove it from code completely?
|
||||
|
||||
|
||||
IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGUMENTS,
|
||||
{{"default", ParallelReplicasCustomKeyFilterType::DEFAULT},
|
||||
|
||||
@ -16,8 +16,10 @@ enum class LoadBalancing
|
||||
/// among replicas with a minimum number of errors selected randomly
|
||||
RANDOM = 0,
|
||||
/// a replica is selected among the replicas with the minimum number of errors
|
||||
/// with the minimum number of distinguished characters in the replica name and local hostname
|
||||
/// with the minimum number of distinguished characters in the replica name prefix and local hostname prefix
|
||||
NEAREST_HOSTNAME,
|
||||
/// just like NEAREST_HOSTNAME, but it count distinguished characters in a levenshtein distance manner
|
||||
HOSTNAME_LEVENSHTEIN_DISTANCE,
|
||||
// replicas with the same number of errors are accessed in the same order
|
||||
// as they are specified in the configuration.
|
||||
IN_ORDER,
|
||||
@ -207,7 +209,6 @@ enum class Dialect
|
||||
{
|
||||
clickhouse,
|
||||
kusto,
|
||||
kusto_auto,
|
||||
prql,
|
||||
};
|
||||
|
||||
|
||||
@ -45,7 +45,6 @@ public:
|
||||
String doGetName() const override;
|
||||
String getNameWithoutVersion() const;
|
||||
const char * getFamilyName() const override { return "AggregateFunction"; }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
TypeIndex getTypeId() const override { return TypeIndex::AggregateFunction; }
|
||||
|
||||
Array getParameters() const { return parameters; }
|
||||
|
||||
@ -35,10 +35,6 @@ public:
|
||||
{
|
||||
return "Array";
|
||||
}
|
||||
String getSQLCompatibleName() const override
|
||||
{
|
||||
return "TEXT";
|
||||
}
|
||||
|
||||
bool canBeInsideNullable() const override
|
||||
{
|
||||
|
||||
@ -13,7 +13,6 @@ public:
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Date; }
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String getSQLCompatibleName() const override { return "DATE"; }
|
||||
|
||||
bool canBeUsedAsVersion() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
@ -13,7 +13,6 @@ public:
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Date32; }
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String getSQLCompatibleName() const override { return "DATE"; }
|
||||
|
||||
Field getDefault() const override
|
||||
{
|
||||
|
||||
@ -38,7 +38,6 @@ public:
|
||||
static constexpr auto family_name = "DateTime";
|
||||
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String getSQLCompatibleName() const override { return "DATETIME"; }
|
||||
String doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeIndex::DateTime; }
|
||||
|
||||
|
||||
@ -28,7 +28,6 @@ public:
|
||||
DataTypeDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_info);
|
||||
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String getSQLCompatibleName() const override { return "DATETIME"; }
|
||||
std::string doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
|
||||
@ -36,30 +36,6 @@ const char * DataTypeEnum<Type>::getFamilyName() const
|
||||
return EnumName<FieldType>::value;
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
std::string DataTypeEnum<Type>::generateMySQLName(const Values & values)
|
||||
{
|
||||
WriteBufferFromOwnString out;
|
||||
|
||||
writeString("ENUM", out);
|
||||
writeChar('(', out);
|
||||
|
||||
auto first = true;
|
||||
for (const auto & name_and_value : values)
|
||||
{
|
||||
if (!first)
|
||||
writeString(", ", out);
|
||||
|
||||
first = false;
|
||||
|
||||
writeQuotedString(name_and_value.first, out);
|
||||
}
|
||||
|
||||
writeChar(')', out);
|
||||
|
||||
return out.str();
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
std::string DataTypeEnum<Type>::generateName(const Values & values)
|
||||
{
|
||||
|
||||
@ -46,14 +46,12 @@ public:
|
||||
private:
|
||||
std::string type_name;
|
||||
static std::string generateName(const Values & values);
|
||||
static std::string generateMySQLName(const Values & values);
|
||||
|
||||
public:
|
||||
explicit DataTypeEnum(const Values & values_);
|
||||
|
||||
std::string doGetName() const override { return type_name; }
|
||||
const char * getFamilyName() const override;
|
||||
String getSQLCompatibleName() const override { return generateMySQLName(this->getValues()); }
|
||||
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
|
||||
@ -42,8 +42,6 @@ public:
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
const char * getFamilyName() const override { return "FixedString"; }
|
||||
/// Use TEXT for compatibility with MySQL to allow arbitrary bytes.
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
|
||||
size_t getN() const
|
||||
{
|
||||
|
||||
@ -24,7 +24,6 @@ public:
|
||||
|
||||
std::string doGetName() const override;
|
||||
const char * getFamilyName() const override { return "Function"; }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Function; }
|
||||
|
||||
const DataTypes & getArgumentTypes() const
|
||||
|
||||
@ -19,7 +19,6 @@ public:
|
||||
static constexpr auto type_id = TypeToTypeIndex<IPv4>;
|
||||
|
||||
const char * getFamilyName() const override { return TypeName<IPv4>.data(); }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
@ -61,7 +60,6 @@ public:
|
||||
static constexpr auto type_id = TypeToTypeIndex<IPv6>;
|
||||
|
||||
const char * getFamilyName() const override { return TypeName<IPv6>.data(); }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
|
||||
@ -27,7 +27,6 @@ public:
|
||||
SerializationPtr doGetDefaultSerialization() const override;
|
||||
std::string doGetName() const override { return fmt::format("Interval{}", kind.toString()); }
|
||||
const char * getFamilyName() const override { return "Interval"; }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Interval; }
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
@ -176,4 +176,8 @@ DataTypePtr removeLowCardinality(const DataTypePtr & type)
|
||||
return type;
|
||||
}
|
||||
|
||||
DataTypePtr removeLowCardinalityAndNullable(const DataTypePtr & type)
|
||||
{
|
||||
return removeNullable(removeLowCardinality(type));
|
||||
};
|
||||
}
|
||||
|
||||
@ -23,7 +23,6 @@ public:
|
||||
return "LowCardinality(" + dictionary_type->getName() + ")";
|
||||
}
|
||||
const char * getFamilyName() const override { return "LowCardinality"; }
|
||||
String getSQLCompatibleName() const override { return dictionary_type->getSQLCompatibleName(); }
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::LowCardinality; }
|
||||
|
||||
@ -92,4 +91,7 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column);
|
||||
/// Convert column of type from_type to type to_type by converting nested LowCardinality columns.
|
||||
ColumnPtr recursiveLowCardinalityTypeConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type);
|
||||
|
||||
/// Removes LowCardinality and Nullable in a correct order and returns T
|
||||
/// if the type is LowCardinality(T) or LowCardinality(Nullable(T)); type otherwise
|
||||
DataTypePtr removeLowCardinalityAndNullable(const DataTypePtr & type);
|
||||
}
|
||||
|
||||
@ -31,7 +31,6 @@ public:
|
||||
std::string doGetName() const override;
|
||||
std::string doGetPrettyName(size_t indent) const override;
|
||||
const char * getFamilyName() const override { return "Map"; }
|
||||
String getSQLCompatibleName() const override { return "JSON"; }
|
||||
|
||||
bool canBeInsideNullable() const override { return false; }
|
||||
|
||||
|
||||
@ -16,7 +16,6 @@ public:
|
||||
static constexpr bool is_parametric = false;
|
||||
|
||||
const char * getFamilyName() const override { return "Nothing"; }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Nothing; }
|
||||
|
||||
|
||||
@ -16,7 +16,6 @@ public:
|
||||
explicit DataTypeNullable(const DataTypePtr & nested_data_type_);
|
||||
std::string doGetName() const override { return "Nullable(" + nested_data_type->getName() + ")"; }
|
||||
const char * getFamilyName() const override { return "Nullable"; }
|
||||
String getSQLCompatibleName() const override { return nested_data_type->getSQLCompatibleName(); }
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Nullable; }
|
||||
|
||||
MutableColumnPtr createColumn() const override;
|
||||
|
||||
@ -11,34 +11,6 @@ Field DataTypeNumberBase<T>::getDefault() const
|
||||
{
|
||||
return NearestFieldType<FieldType>();
|
||||
}
|
||||
template <typename T>
|
||||
String DataTypeNumberBase<T>::getSQLCompatibleName() const
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Int8>)
|
||||
return "TINYINT";
|
||||
else if constexpr (std::is_same_v<T, Int16>)
|
||||
return "SMALLINT";
|
||||
else if constexpr (std::is_same_v<T, Int32>)
|
||||
return "INTEGER";
|
||||
else if constexpr (std::is_same_v<T, Int64>)
|
||||
return "BIGINT";
|
||||
else if constexpr (std::is_same_v<T, UInt8>)
|
||||
return "TINYINT UNSIGNED";
|
||||
else if constexpr (std::is_same_v<T, UInt16>)
|
||||
return "SMALLINT UNSIGNED";
|
||||
else if constexpr (std::is_same_v<T, UInt32>)
|
||||
return "INTEGER UNSIGNED";
|
||||
else if constexpr (std::is_same_v<T, UInt64>)
|
||||
return "BIGINT UNSIGNED";
|
||||
else if constexpr (std::is_same_v<T, Float32>)
|
||||
return "FLOAT";
|
||||
else if constexpr (std::is_same_v<T, Float64>)
|
||||
return "DOUBLE";
|
||||
/// Unsupported types are converted to TEXT
|
||||
else
|
||||
return "TEXT";
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MutableColumnPtr DataTypeNumberBase<T>::createColumn() const
|
||||
{
|
||||
|
||||
@ -25,7 +25,6 @@ public:
|
||||
using ColumnType = ColumnVector<T>;
|
||||
|
||||
const char * getFamilyName() const override { return TypeName<T>.data(); }
|
||||
String getSQLCompatibleName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeToTypeIndex<T>; }
|
||||
|
||||
Field getDefault() const override;
|
||||
|
||||
@ -23,7 +23,6 @@ public:
|
||||
DataTypeObject(const String & schema_format_, bool is_nullable_);
|
||||
|
||||
const char * getFamilyName() const override { return "Object"; }
|
||||
String getSQLCompatibleName() const override { return "JSON"; }
|
||||
String doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Object; }
|
||||
|
||||
|
||||
@ -15,7 +15,6 @@ class DataTypeSet final : public IDataTypeDummy
|
||||
public:
|
||||
static constexpr bool is_parametric = true;
|
||||
const char * getFamilyName() const override { return "Set"; }
|
||||
String getSQLCompatibleName() const override { return "TEXT"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Set; }
|
||||
bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); }
|
||||
|
||||
@ -21,8 +21,6 @@ public:
|
||||
return "String";
|
||||
}
|
||||
|
||||
String getSQLCompatibleName() const override { return "BLOB"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
MutableColumnPtr createColumn() const override;
|
||||
|
||||
@ -34,7 +34,6 @@ public:
|
||||
std::string doGetName() const override;
|
||||
std::string doGetPrettyName(size_t indent) const override;
|
||||
const char * getFamilyName() const override { return "Tuple"; }
|
||||
String getSQLCompatibleName() const override { return "JSON"; }
|
||||
|
||||
bool canBeInsideNullable() const override { return false; }
|
||||
bool supportsSparseSerialization() const override { return true; }
|
||||
|
||||
@ -18,7 +18,6 @@ public:
|
||||
static constexpr auto type_id = TypeIndex::UUID;
|
||||
|
||||
const char * getFamilyName() const override { return "UUID"; }
|
||||
String getSQLCompatibleName() const override { return "CHAR"; }
|
||||
|
||||
TypeIndex getTypeId() const override { return type_id; }
|
||||
|
||||
|
||||
@ -28,18 +28,6 @@ std::string DataTypeDecimal<T>::doGetName() const
|
||||
return fmt::format("Decimal({}, {})", this->precision, this->scale);
|
||||
}
|
||||
|
||||
template <is_decimal T>
|
||||
std::string DataTypeDecimal<T>::getSQLCompatibleName() const
|
||||
{
|
||||
/// See https://dev.mysql.com/doc/refman/8.0/en/precision-math-decimal-characteristics.html
|
||||
/// DECIMAL(M,D)
|
||||
/// M is the maximum number of digits (the precision). It has a range of 1 to 65.
|
||||
/// D is the number of digits to the right of the decimal point (the scale). It has a range of 0 to 30 and must be no larger than M.
|
||||
if (this->precision > 65 || this->scale > 30)
|
||||
return "TEXT";
|
||||
return fmt::format("DECIMAL({}, {})", this->precision, this->scale);
|
||||
}
|
||||
|
||||
template <is_decimal T>
|
||||
bool DataTypeDecimal<T>::equals(const IDataType & rhs) const
|
||||
{
|
||||
|
||||
@ -39,7 +39,6 @@ public:
|
||||
static constexpr auto family_name = "Decimal";
|
||||
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
String getSQLCompatibleName() const override;
|
||||
|
||||
std::string doGetName() const override;
|
||||
TypeIndex getTypeId() const override { return TypeToTypeIndex<T>; }
|
||||
|
||||
@ -83,8 +83,6 @@ public:
|
||||
|
||||
/// Name of data type family (example: FixedString, Array).
|
||||
virtual const char * getFamilyName() const = 0;
|
||||
/// Name of corresponding data type in MySQL (exampe: Bigint, Blob, etc)
|
||||
virtual String getSQLCompatibleName() const = 0;
|
||||
|
||||
/// Data type id. It's used for runtime type checks.
|
||||
virtual TypeIndex getTypeId() const = 0;
|
||||
|
||||
@ -339,7 +339,7 @@ static DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool che
|
||||
for (size_t i = 1; i < subtypes.size(); ++i)
|
||||
if (first_dim != getNumberOfDimensions(*subtypes[i]))
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH,
|
||||
"Uncompatible types of subcolumn '{}': {} and {}",
|
||||
"Incompatible types of subcolumn '{}': {} and {}",
|
||||
key.getPath(), subtypes[0]->getName(), subtypes[i]->getName());
|
||||
|
||||
tuple_paths.emplace_back(key);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user