From c0732ddc12cca6d7a0f1a20a1278434befe3a753 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 08:18:38 +0000 Subject: [PATCH 1/4] Fix datetime with timezone --- programs/odbc-bridge/ODBCBlockInputStream.cpp | 2 +- src/DataStreams/PostgreSQLBlockInputStream.cpp | 6 +++--- src/Formats/MySQLBlockInputStream.cpp | 2 +- tests/integration/test_storage_postgresql/test.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCBlockInputStream.cpp index b23d09e0481..25c953c0b71 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -132,7 +132,7 @@ void ODBCBlockInputStream::insertValue( auto value = row.get(idx); ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index a41280847a5..0df974b2405 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -170,7 +170,7 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view { ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); @@ -272,11 +272,11 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [](std::string & field) -> Field + parser = [data_type](std::string & field) -> Field { ReadBufferFromString in(field); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); return time; }; else if (which.isDecimal32()) diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 3ea86c82fa3..91e51de4603 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -169,7 +169,7 @@ namespace { ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type).getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index f81033822c8..716f16c6211 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -308,6 +308,21 @@ def test_postgres_distributed(started_cluster): assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') +def test_datetime_with_timezone(started_cluster): + conn = get_postgres_conn(started_cluster, started_cluster.postgres_ip, True) + cursor = conn.cursor() + cursor.execute("CREATE TABLE test_timezone (ts timestamp without time zone, ts_z timestamp with time zone)") + cursor.execute("insert into test_timezone select '2014-04-04 20:00:00', '2014-04-04 20:00:00'::timestamptz at time zone 'America/New_York';") + cursor.execute("select * from test_timezone") + result = cursor.fetchall()[0] + print(result[0], str(result[1])[:-6]) + node1.query("create table test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test_timezone', 'postgres', 'mysecretpassword');") + assert(node1.query("select ts from test_timezone").strip() == str(result[0])) + # [:-6] because 2014-04-04 16:00:00+00:00 -> 2014-04-04 16:00:00 + assert(node1.query("select ts_z from test_timezone").strip() == str(result[1])[:-6]) + assert(node1.query("select * from test_timezone") == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n") + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From fb9c92cb6c702b02ce20577b03394f2fa9cc6498 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 10:32:40 +0000 Subject: [PATCH 2/4] Fix --- src/DataStreams/PostgreSQLBlockInputStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index 0df974b2405..a486df83025 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -272,11 +272,11 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [data_type](std::string & field) -> Field + parser = [nested](std::string & field) -> Field { ReadBufferFromString in(field); time_t time = 0; - readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); + readDateTimeText(time, in, assert_cast(nested.get())->getTimeZone()); return time; }; else if (which.isDecimal32()) From 2487063be3d9079e8ae86d32dbf7ad0be2ee2e36 Mon Sep 17 00:00:00 2001 From: Evgeniy Gatov Date: Mon, 21 Jun 2021 22:23:13 +0300 Subject: [PATCH 3/4] METR-41529 --- base/mysqlxx/Query.cpp | 2 +- base/mysqlxx/ya.make | 39 +++++++++++++++++++++++++++++++++++++++ base/mysqlxx/ya.make.in | 28 ++++++++++++++++++++++++++++ base/ya.make | 1 + 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 base/mysqlxx/ya.make create mode 100644 base/mysqlxx/ya.make.in diff --git a/base/mysqlxx/Query.cpp b/base/mysqlxx/Query.cpp index d4514c3e988..c0d5c20fdfd 100644 --- a/base/mysqlxx/Query.cpp +++ b/base/mysqlxx/Query.cpp @@ -2,7 +2,7 @@ #include #include #else -#include +#include //Y_IGNORE #include #endif diff --git a/base/mysqlxx/ya.make b/base/mysqlxx/ya.make new file mode 100644 index 00000000000..aabc9922e72 --- /dev/null +++ b/base/mysqlxx/ya.make @@ -0,0 +1,39 @@ +# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it. +LIBRARY() + +OWNER(g:clickhouse) + +CFLAGS(-g0) + +PEERDIR( + contrib/restricted/boost/libs + contrib/libs/libmysql_r + contrib/libs/poco/Foundation + contrib/libs/poco/Util +) + +ADDINCL( + GLOBAL clickhouse/base + clickhouse/base + contrib/libs/libmysql_r +) + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +SRCS( + Connection.cpp + Exception.cpp + Pool.cpp + PoolFactory.cpp + PoolWithFailover.cpp + Query.cpp + ResultBase.cpp + Row.cpp + UseQueryResult.cpp + Value.cpp + +) + +END() diff --git a/base/mysqlxx/ya.make.in b/base/mysqlxx/ya.make.in new file mode 100644 index 00000000000..10755078e20 --- /dev/null +++ b/base/mysqlxx/ya.make.in @@ -0,0 +1,28 @@ +LIBRARY() + +OWNER(g:clickhouse) + +CFLAGS(-g0) + +PEERDIR( + contrib/restricted/boost/libs + contrib/libs/libmysql_r + contrib/libs/poco/Foundation + contrib/libs/poco/Util +) + +ADDINCL( + GLOBAL clickhouse/base + clickhouse/base + contrib/libs/libmysql_r +) + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +SRCS( + +) + +END() diff --git a/base/ya.make b/base/ya.make index 9f4cf0fd4a7..19a16044280 100644 --- a/base/ya.make +++ b/base/ya.make @@ -4,6 +4,7 @@ RECURSE( common daemon loggers + mysqlxx pcg-random widechar_width readpassphrase From cbc7e61140948fc591d748437208a3001d003161 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 22 Jun 2021 02:23:53 +0300 Subject: [PATCH 4/4] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6e827c3c240..f146913a2e8 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -246,3 +246,4 @@ 01901_test_attach_partition_from 01910_view_dictionary 01824_prefer_global_in_and_join +01576_alias_column_rewrite