thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into localhost

Browse Source
This commit is contained in:
proller 2017-01-25 15:38:46 +03:00
commit 1b7054e15b
398 changed files with 10378 additions and 5295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
CMakeLists.txt
View File

@ -50,7 +50,7 @@ if (NOT AARCH64)
set (MACHINE_FLAGS "-msse4 -mpopcnt")
endif ()
set (COMMON_WARNING_FLAGS "-Wall -Werror")
set (COMMON_WARNING_FLAGS "-Wall") # -Werror is also added inside directories with our own code.
set (CXX_WARNING_FLAGS "-Wnon-virtual-dtor")
set (CXX11_ABI "ENABLE" CACHE STRING "Use C++11 ABI: DEFAULT, ENABLE, DISABLE")
@ -62,8 +62,6 @@ if (NOT $ENV{USE_STATIC_LIBRARIES})
set (USE_STATIC_LIBRARIES FALSE)
endif ()
set (USE_INTERNAL_BOOST_LIBRARY TRUE CACHE BOOL "Set to FALSE to use system boost library instead of bundled")
if (NOT $ENV{USE_INTERNAL_BOOST_LIBRARY})
set (USE_INTERNAL_BOOST_LIBRARY FALSE)
@ -76,10 +74,6 @@ endif ()
option (USE_INTERNAL_POCO_LIBRARY "Set to FALSE to use system poco library instead of bundled" ON)
option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ON)
option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Note that it is not compatible with ASan." OFF)
if ($ENV{GLIBC_COMPATIBILITY})
set (GLIBC_COMPATIBILITY TRUE)
endif ()
option (ENABLE_LIBTCMALLOC "Set to TRUE to enable libtcmalloc." ON)
if (NOT $ENV{ENABLE_LIBTCMALLOC})
@ -91,6 +85,11 @@ if ($ENV{DEBUG_LIBTCMALLOC})
set (DEBUG_LIBTCMALLOC TRUE)
endif ()
option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Note that it is not compatible with ASan." OFF)
if ($ENV{GLIBC_COMPATIBILITY})
set (GLIBC_COMPATIBILITY TRUE)
endif ()
if (GLIBC_COMPATIBILITY)
set (GLIBC_COMPATIBILITY_COMPILE_FLAGS "-include ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/common/glibc_compatibility.h")
set (GLIBC_COMPATIBILITY_LINK_FLAGS "-Wl,--wrap=memcpy")
@ -104,16 +103,24 @@ else ()
set (CXX11_ABI_FLAGS "")
endif ()
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++1y ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CXX11_ABI_FLAGS}")
#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${CXX11_ABI_FLAGS}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CXX11_ABI_FLAGS}")
#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
if (PIPE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
endif ()
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} -std=gnu++1y ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMPILER_FLAGS}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COMPILER_FLAGS} -O3")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMPILER_FLAGS} -O0 -g3 -ggdb3 -fno-inline")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${MACHINE_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMPILER_FLAGS}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${COMPILER_FLAGS} -O3")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMPILER_FLAGS} -O0 -g3 -ggdb3 -fno-inline")
if (NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_SYSTEM MATCHES "FreeBSD"))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@ -123,16 +130,31 @@ if (NOT APPLE)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GLIBC_COMPATIBILITY_LINK_FLAGS} ${CXX11_ABI_FLAGS}")
endif ()
# -fuse-ld=gold - fix linkage for gcc-5.4, gcc-6.1
# See more in http://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1
set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} -O3 -g -fsanitize=address -fno-omit-frame-pointer -fuse-ld=gold ${CXX11_ABI_FLAGS}")
set (CMAKE_CXX_FLAGS_UBSAN "${CMAKE_CXX_FLAGS_UBSAN} -O3 -g -fsanitize=undefined -fno-omit-frame-pointer ${CXX11_ABI_FLAGS}")
set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} -O3 -g -fsanitize=address -fno-omit-frame-pointer -fuse-ld=gold ${CXX11_ABI_FLAGS}")
set (CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_UBSAN} -O3 -g -fsanitize=undefined -fno-omit-frame-pointer ${CXX11_ABI_FLAGS}")
if (USE_STATIC_LIBRARIES AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -no-pie")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -no-pie")
endif ()
set (SAN_FLAGS "-O3 -g -fno-omit-frame-pointer")
set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address")
set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address")
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
# -fuse-ld=gold - fix linkage for gcc-5.4, gcc-6.1
# See more in http://stackoverflow.com/questions/37603238/fsanitize-not-using-gold-linker-in-gcc-6-1
set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} -fuse-ld=gold")
set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} -fuse-ld=gold")
endif ()
set (CMAKE_CXX_FLAGS_UBSAN "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
set (CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined")
set (CMAKE_CXX_FLAGS_MSAN "${CMAKE_CXX_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory")
set (CMAKE_C_FLAGS_MSAN "${CMAKE_C_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory")
set (CMAKE_CXX_FLAGS_TSAN "${CMAKE_CXX_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread")
set (CMAKE_C_FLAGS_TSAN "${CMAKE_C_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread")
# Flags for test coverage
if (TEST_COVERAGE)
set (CMAKE_CXX_FLAGS_DEBUG "-O0 -g -fprofile-arcs -ftest-coverage -fPIC -DIS_DEBUG ${CXX11_ABI_FLAGS}")
set (CMAKE_CXX_FLAGS_DEBUG "${COMPILER_FLAGS} -O0 -g -fprofile-arcs -ftest-coverage -fPIC -DIS_DEBUG")
endif (TEST_COVERAGE)
# Run tests with "make check"
@ -149,33 +171,10 @@ else ()
set (CLICKHOUSE_ETC_DIR ${CMAKE_INSTALL_PREFIX}/etc)
endif ()
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdouble-conversion/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libsparsehash/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libre2/)
include_directories (BEFORE ${ClickHouse_BINARY_DIR}/contrib/libre2/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/)
include_directories (BEFORE ${ClickHouse_BINARY_DIR}/libs/libcommon/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libpocoext/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libmysqlxx/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libzkutil/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/dbms/include)
include (cmake/find_openssl.cmake)
include (cmake/find_icu4c.cmake)
include (cmake/find_boost.cmake)
include (cmake/find_poco.cmake)
include (cmake/find_libtool.cmake)
include (cmake/find_mysqlclient.cmake)
include (cmake/find_rt.cmake)
if (ENABLE_LIBTCMALLOC)
include (cmake/find_gperftools.cmake)
@ -183,17 +182,17 @@ endif ()
# Directory for Yandex specific files
set (CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)
if (EXISTS ${CLICKHOUSE_PRIVATE_DIR})
add_subdirectory (${CLICKHOUSE_PRIVATE_DIR})
endif ()
add_subdirectory (contrib)
add_subdirectory (libs)
add_subdirectory (utils)
add_subdirectory (dbms)
if (EXISTS ${CLICKHOUSE_PRIVATE_DIR})
add_subdirectory (private)
endif ()
message (STATUS "C_FLAGS: =${CMAKE_C_FLAGS}")
message (STATUS "CXX_FLAGS:=${CMAKE_CXX_FLAGS}")
message (STATUS "C_FLAGS = ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}")
message (STATUS "CXX_FLAGS = ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
message (STATUS "LINK_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
include (cmake/print_include_directories.cmake)

2
cmake/create_init_script.cmake
View File

@ -6,7 +6,7 @@
# Будет создан init.d скрипт с названием divider для демона (бинарника) Divider
macro (create_init_script daemonname)
set (filename ${daemonname}-metrika-yandex)
set (filename ${daemonname})
# Опционально принимаем filename вторым аргументом.
set (extra_args ${ARGN})
list (LENGTH extra_args num_extra_args)

14
cmake/dbms_include.cmake Normal file
View File

@ -0,0 +1,14 @@
include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
# TODO:
# move code with incldes from .h to .cpp and clean this list:
include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
# for generated config_version.h and config_common.h:
include_directories (${CMAKE_BINARY_DIR}/libs/libcommon/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libpocoext/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libzkutil/include)
include_directories (${CMAKE_SOURCE_DIR}/libs/libmysqlxx/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libzookeeper/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libcityhash/include)
include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libdouble-conversion)

53
cmake/find_glib.cmake
View File

@ -1,53 +0,0 @@
set (GLIB_HINTS "/usr/local/opt/glib/lib")
set (GLIB_INCLUDE_HINTS
"/usr/local/include/glib-2.0/"
"/usr/local/lib/glib-2.0/include/"
"/usr/include/glib-2.0/"
"/usr/lib64/glib-2.0/include/")
if (AARCH64)
set (GLIB_INCLUDE_HINTS ${GLIB_INCLUDE_HINTS} "/usr/lib/aarch64-linux-gnu/glib-2.0/include/")
else ()
set (GLIB_INCLUDE_HINTS ${GLIB_INCLUDE_HINTS} "/usr/lib/x86_64-linux-gnu/glib-2.0/include/")
endif ()
if (USE_STATIC_LIBRARIES)
find_library (GLIB_LIB libglib-2.0.a HINTS ${GLIB_HINTS})
else ()
find_library (GLIB_LIB glib-2.0 HINTS ${GLIB_HINTS})
endif ()
find_path (GLIB_INCLUDE_DIR NAMES glib.h HINTS ${GLIB_INCLUDE_HINTS})
find_path (GLIB_CONFIG_INCLUDE_DIR NAMES glibconfig.h HINTS ${GLIB_INCLUDE_HINTS})
include_directories (${GLIB_INCLUDE_DIR})
include_directories (${GLIB_CONFIG_INCLUDE_DIR})
if (APPLE)
set (INTL_HINTS "/usr/local/opt/gettext/lib")
if (USE_STATIC_LIBRARIES)
find_library (INTL_LIB libintl.a HINTS ${INTL_HINTS})
else ()
find_library (INTL_LIB intl HINTS ${INTL_HINTS})
endif ()
set (ICONV_HINTS "/usr/local/opt/libiconv/lib")
if (USE_STATIC_LIBRARIES)
find_library (ICONV_LIB libiconv.a HINTS ${ICONV_HINTS})
else ()
find_library (ICONV_LIB iconv HINTS ${ICONV_HINTS})
endif ()
find_library (CORE_FOUNDATION_LIB CoreFoundation)
find_library (CARBON_LIB Carbon)
set (GLIB_LIBS ${GLIB_LIB} ${INTL_LIB} ${ICONV_LIB} ${CORE_FOUNDATION_LIB} ${CARBON_LIB})
elseif (CMAKE_SYSTEM MATCHES "FreeBSD")
set (ICONV_HINTS "")
set (INTL_HINTS "")
if (USE_STATIC_LIBRARIES)
find_library (ICONV_LIB libiconv.a HINTS ${ICONV_HINTS})
find_library (INTL_LIB libintl.a HINTS ${INTL_HINTS})
else ()
find_library (ICONV_LIB iconv HINTS ${ICONV_HINTS})
find_library (INTL_LIB intl HINTS ${INTL_HINTS})
endif ()
set (GLIB_LIBS ${GLIB_LIB} ${INTL_LIB} ${ICONV_LIB})
else ()
set (GLIB_LIBS ${GLIB_LIB})
endif ()

1
cmake/find_gperftools.cmake
View File

@ -8,4 +8,3 @@ else ()
endif ()
message(STATUS "Using gperftools: ${GPERFTOOLS_INCLUDE_DIR} : ${GPERFTOOLS_TCMALLOC}")

20
cmake/find_icu4c.cmake
View File

@ -1,12 +1,16 @@
include_directories ("/usr/local/opt/icu4c/include")
set (ICU_HINTS "/usr/local/opt/icu4c/lib")
set (ICU_PATHS "/usr/local/opt/icu4c/lib")
set (ICU_INCLUDE_PATHS "/usr/local/opt/icu4c/include")
if (USE_STATIC_LIBRARIES)
find_library (ICUI18N libicui18n.a HINTS ${ICU_HINTS})
find_library (ICUUC libicuuc.a HINTS ${ICU_HINTS})
find_library (ICUDATA libicudata.a HINTS ${ICU_HINTS})
find_library (ICUI18N libicui18n.a PATHS ${ICU_PATHS})
find_library (ICUUC libicuuc.a PATHS ${ICU_PATHS})
find_library (ICUDATA libicudata.a PATHS ${ICU_PATHS})
else ()
find_library (ICUI18N icui18n HINTS ${ICU_HINTS})
find_library (ICUUC icuuc HINTS ${ICU_HINTS})
find_library (ICUDATA icudata HINTS ${ICU_HINTS})
find_library (ICUI18N icui18n PATHS ${ICU_PATHS})
find_library (ICUUC icuuc PATHS ${ICU_PATHS})
find_library (ICUDATA icudata PATHS ${ICU_PATHS})
endif ()
set (ICU_LIBS ${ICUI18N} ${ICUUC} ${ICUDATA})
find_path (ICU_INCLUDE_DIR NAMES unicode/unistr.h PATHS ${ICU_INCLUDE_PATHS})
message(STATUS "Using icu: ${ICU_INCLUDE_DIR} : ${ICU_LIBS}")
include_directories (${ICU_INCLUDE_DIR})

8
cmake/find_poco.cmake
View File

@ -3,16 +3,11 @@ if (USE_INTERNAL_POCO_LIBRARY)
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Foundation/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Util/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Net/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/NetSSL_OpenSSL/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Data/include/"
#"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Data/MySQL/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Data/ODBC/include/"
#"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Data/SQLite/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Crypto/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/XML/include/"
#"${ClickHouse_SOURCE_DIR}/contrib/libpoco/JSON/include/"
"${ClickHouse_SOURCE_DIR}/contrib/libpoco/MongoDB/include/"
#"${ClickHouse_SOURCE_DIR}/contrib/libpoco/Zip/include/"
)
if (USE_STATIC_LIBRARIES)
@ -22,7 +17,6 @@ if (USE_INTERNAL_POCO_LIBRARY)
set (Poco_Net_LIBRARY PocoNet)
set (Poco_Util_LIBRARY PocoUtil)
set (Poco_XML_LIBRARY PocoXML)
set (Poco_NetSSL_OpenSSL_LIBRARY PocoNetSSL_OpenSSL)
set (Poco_Data_LIBRARY PocoData)
set (Poco_Crypto_LIBRARY PocoCrypto)
set (Poco_DataODBC_LIBRARY PocoDataODBC)
@ -30,7 +24,7 @@ if (USE_INTERNAL_POCO_LIBRARY)
set (Poco_Foundation_LIBRARY PocoFoundation)
include_directories (BEFORE ${Poco_INCLUDE_DIRS})
else ()
find_package (Poco REQUIRED Util Net XML NetSSL_OpenSSL Data Crypto DataODBC MongoDB Foundation)
find_package (Poco REQUIRED Util Net XML Data Crypto DataODBC MongoDB Foundation)
include_directories (${Poco_INCLUDE_DIRS})
endif ()

44
cmake/find_readline_edit.cmake Normal file
View File

@ -0,0 +1,44 @@
if (NOT READLINE_PATHS)
set (READLINE_PATHS "/usr/local/opt/readline/lib")
if (USE_STATIC_LIBRARIES)
find_library (READLINE_LIB NAMES libreadline.a PATHS ${READLINE_PATHS})
else ()
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
endif ()
if (USE_STATIC_LIBRARIES)
find_library (TERMCAP_LIB NAMES libtermcap.a termcap)
else ()
find_library (TERMCAP_LIB NAMES termcap)
endif ()
if (USE_STATIC_LIBRARIES)
find_library (EDIT_LIB NAMES libedit.a)
else ()
find_library (EDIT_LIB NAMES edit)
endif ()
set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
if (READLINE_LIB)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
add_definitions (-D USE_READLINE)
set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIB})
message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
elseif (EDIT_LIB)
if (USE_STATIC_LIBRARIES)
find_library (CURSES_LIB NAMES libcurses.a)
else ()
find_library (CURSES_LIB NAMES curses)
endif ()
add_definitions (-D USE_LIBEDIT)
find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIB})
message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
else ()
message (STATUS "Not using any library for line editing.")
endif ()
if (READLINE_INCLUDE_DIR)
include_directories (${READLINE_INCLUDE_DIR})
endif ()
endif ()

2
cmake/print_include_directories.cmake
View File

@ -1,4 +1,4 @@
get_property (dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
get_property (dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/dbms PROPERTY INCLUDE_DIRECTORIES)
file (WRITE ${CMAKE_CURRENT_BINARY_DIR}/include_directories.txt "")
foreach (dir ${dirs})
string (REPLACE "${ClickHouse_SOURCE_DIR}" "." dir "${dir}")

1
contrib/CMakeLists.txt vendored
View File

@ -17,6 +17,7 @@ add_subdirectory (libcityhash)
add_subdirectory (libfarmhash)
add_subdirectory (libmetrohash)
add_subdirectory (libzlib-ng)
add_subdirectory (libcctz)
if (ENABLE_LIBTCMALLOC AND USE_INTERNAL_GPERFTOOLS_LIBRARY)
add_subdirectory (libtcmalloc)

21
contrib/libcctz/CMakeLists.txt Normal file
View File

@ -0,0 +1,21 @@
include_directories (include)
add_library(cctz
src/time_zone_libc.cc
src/time_zone_posix.cc
src/time_zone_lookup.cc
src/time_zone_info.cc
src/time_zone_if.cc
src/time_zone_format.cc
src/time_zone_impl.cc
src/time_zone_libc.h
src/time_zone_if.h
src/tzfile.h
src/time_zone_impl.h
src/time_zone_posix.h
src/time_zone_info.h
include/time_zone.h
include/civil_time_detail.h
include/civil_time.h)

202
contrib/libcctz/LICENSE.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

192
contrib/libcctz/README.md Normal file
View File

@ -0,0 +1,192 @@
This is not an official Google product.
# Overview
CCTZ contains two libraries that cooperate with `<chrono>` to give C++
programmers all the necessary tools for computing with dates, times, and time
zones in a simple and correct manner. The libraries in CCTZ are:
* **The Civil-Time Library** &mdash; This is a header-only library that supports
computing with human-scale time, such as dates (which are represented by the
`cctz::civil_day` class). This library is declared in [`include/civil_time.h`]
(https://github.com/google/cctz/blob/master/include/civil_time.h).
* **The Time-Zone Library** &mdash; This library uses the IANA time zone
database that is installed on the system to convert between *absolute time*
and *civil time*. This library is declared in [`include/time_zone.h`](https://github.com/google/cctz/blob/master/include/time_zone.h).
These libraries are currently known to work on **Linux** and **Mac OS X**. We
are actively interested in help getting them working on Windows. Please contact
us if you're interested in contributing.
# Getting Started
CCTZ is best built and tested using the [Bazel](http://bazel.io) build system
and the [Google Test](https://github.com/google/googletest) framework. (There
is also a simple [`Makefile`](https://github.com/google/cctz/blob/master/Makefile)
that should work if you're unable to use Bazel.)
1. Download/install Bazel http://bazel.io/docs/install.html
2. Get the cctz source: `git clone https://github.com/google/cctz.git` then `cd
cctz`
3. Build cctz and run the tests: `bazel test :all`
Note: When using CCTZ in your own project, you might find it easiest to compile
the sources using your existing build system.
Next Steps:
1. See the documentation for the libraries in CCTZ:
* Civil Time: [`include/civil_time.h`](https://github.com/google/cctz/blob/master/include/civil_time.h)
* Time Zone: [`include/time_zone.h`](https://github.com/google/cctz/blob/master/include/time_zone.h)
2. Look at the examples in https://github.com/google/cctz/tree/master/examples
3. Join our mailing list to ask questions and keep informed of changes:
* https://groups.google.com/forum/#!forum/cctz
# Fundamental Concepts
*[The concepts presented here describe general truths about the problem domain
and are library and language agnostic. An understanding of these concepts helps
the programmer correctly reason about even the most-complicated time-programming
challenges and produce the simplest possible solutions.]*
There are two main ways to think about time in a computer program: as *absolute
time*, and as *civil time*. Both have their uses and it is important to
understand when each is appropriate. Absolute and civil times may be converted
back and forth using a *time zone* &mdash; this is the only way to correctly
convert between them. Let us now look more deeply at the three main concepts of
time programming: Absolute Time, Civil Time, and Time Zone.
*Absolute time* uniquely and universally represents a specific instant in time.
It has no notion of calendars, or dates, or times of day. Instead, it is a
measure of the passage of real time, typically as a simple count of ticks since
some epoch. Absolute times are independent of all time zones and do not suffer
from human-imposed complexities such as daylight-saving time (DST). Many C++
types exist to represent absolute times, classically `time_t` and more recently
`std::chrono::time_point`.
*Civil time* is the legally recognized representation of time for ordinary
affairs (cf. http://www.merriam-webster.com/dictionary/civil). It is a
human-scale representation of time that consists of the six fields &mdash;
year, month, day, hour, minute, and second (sometimes shortened to "YMDHMS")
&mdash; and it follows the rules of the Proleptic Gregorian Calendar, with
24-hour days divided into 60-minute hours and 60-second minutes. Like absolute
times, civil times are also independent of all time zones and their related
complexities (e.g., DST). While `std::tm` contains the six civil-time fields
(YMDHMS), plus a few more, it does not have behavior to enforce the rules of
civil time.
*Time zones* are geo-political regions within which human-defined rules are
shared to convert between the absolute-time and civil-time domains. A time
zone's rules include things like the region's offset from the UTC time standard,
daylight-saving adjustments, and short abbreviation strings. Time zones often
have a history of disparate rules that apply only for certain periods, because
the rules may change at the whim of a region's local government. For this
reason, time-zone rules are usually compiled into data snapshots that are used
at runtime to perform conversions between absolute and civil times. There is
currently no C++ standard library supporting arbitrary time zones.
In order for programmers to reason about and program applications that correctly
deal with these concepts, they must have a library that correctly implements the
above concepts. CCTZ adds to the existing C++11 `<chrono>` library to fully
implement the above concepts.
* Absolute time &mdash; This is implemented by the existing C++11
[`<chrono>`](http://en.cppreference.com/w/cpp/chrono)
library without modification. For example, an absolute point in time is
represented by a `std::chrono::time_point`.
* Civil time &mdash; This is implemented by the [`include/civil_time.h`](https://github.com/google/cctz/blob/master/include/civil_time.h) library
that is provided as part of CCTZ. For example, a "date" is represented by a
`cctz::civil_day`.
* Time zone &mdash; This is implemented by the [`include/time_zone.h`](https://github.com/google/cctz/blob/master/include/time_zone.h) library
that is provided as part of CCTZ. For example, a time zone is represented by
an instance of the class `cctz::time_zone`.
# Examples
## Hello February 2016
This "hello world" example uses a for-loop to iterate the days from the first of
February until the month of March. Each day is streamed to output, and if the
day happens to be the 29th, we also output the day of the week.
```
#include <iostream>
#include "civil_time.h"
int main() {
for (cctz::civil_day d(2016, 2, 1); d < cctz::civil_month(2016, 3); ++d) {
std::cout << "Hello " << d;
if (d.day() == 29) {
std::cout << " <- leap day is a " << cctz::get_weekday(d);
}
std::cout << "\n";
}
}
```
The output of the above program is
```
Hello 2016-02-01
Hello 2016-02-02
Hello 2016-02-03
[...]
Hello 2016-02-27
Hello 2016-02-28
Hello 2016-02-29 <- leap day is a Monday
```
## One giant leap
This example shows how to use all three libraries (`<chrono>`, civil time, and
time zone) together. In this example, we know that viewers in New York watched
Neil Armstrong first walk on the moon on July 20, 1969 at 10:56 PM. But we'd
like to see what time it was for our friend watching in Sydney Australia.
```
#include <iostream>
#include "civil_time.h"
#include "time_zone.h"
int main() {
cctz::time_zone nyc;
cctz::load_time_zone("America/New_York", &nyc);
// Converts the input civil time in NYC to an absolute time.
const auto moon_walk =
cctz::convert(cctz::civil_second(1969, 7, 20, 22, 56, 0), nyc);
std::cout << "Moon walk in NYC: "
<< cctz::format("%Y-%m-%d %H:%M:%S %Ez\n", moon_walk, nyc);
cctz::time_zone syd;
if (!cctz::load_time_zone("Australia/Sydney", &syd)) return -1;
std::cout << "Moon walk in SYD: "
<< cctz::format("%Y-%m-%d %H:%M:%S %Ez\n", moon_walk, syd);
}
```
The output of the above program is
```
Moon walk in NYC: 1969-07-20 22:56:00 -04:00
Moon walk in SYD: 1969-07-21 12:56:00 +10:00
```
This example shows that the absolute time (the `std::chrono::time_point`) of the
first walk on the moon is the same no matter the time zone of the viewer (the
same time point is used in both calls to `format()`). The only difference is the
time zone in which the `moon_walk` time point is rendered. And in this case we
can see that our friend in Sydney was probably eating lunch while watching that
historic event.
# References
* CCTZ [FAQ](https://github.com/google/cctz/wiki/FAQ)
* See also the [Time Programming Fundamentals](https://youtu.be/2rnIHsqABfM)
talk from CppCon 2015 ([slides available here](http://goo.gl/ofof4N)). This
talk mostly describes the older CCTZ v1 API, but the *concepts* are the same.
* ISO C++ proposal to standardize the Civil-Time Library:
https://github.com/devjgm/papers/blob/master/d0215r1.md
* ISO C++ proposal to standardize the Time-Zone Library:
https://github.com/devjgm/papers/blob/master/d0216r1.md

325
contrib/libcctz/include/civil_time.h Normal file
View File

@ -0,0 +1,325 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CCTZ_CIVIL_TIME_H_
#define CCTZ_CIVIL_TIME_H_
#include "civil_time_detail.h"
namespace cctz {
// The term "civil time" refers to the legally recognized human-scale time
// that is represented by the six fields YYYY-MM-DD hh:mm:ss. Modern-day civil
// time follows the Gregorian Calendar and is a time-zone-independent concept.
// A "date" is perhaps the most common example of a civil time (represented in
// this library as cctz::civil_day). This library provides six classes and a
// handful of functions that help with rounding, iterating, and arithmetic on
// civil times while avoiding complications like daylight-saving time (DST).
//
// The following six classes form the core of this civil-time library:
//
// * civil_second
// * civil_minute
// * civil_hour
// * civil_day
// * civil_month
// * civil_year
//
// Each class is a simple value type with the same interface for construction
// and the same six accessors for each of the civil fields (year, month, day,
// hour, minute, and second, aka YMDHMS). These classes differ only in their
// alignment, which is indicated by the type name and specifies the field on
// which arithmetic operates.
//
// Each class can be constructed by passing up to six optional integer
// arguments representing the YMDHMS fields (in that order) to the
// constructor. Omitted fields are assigned their minimum valid value. Hours,
// minutes, and seconds will be set to 0, month and day will be set to 1, and
// since there is no minimum valid year, it will be set to 1970. So, a
// default-constructed civil-time object will have YMDHMS fields representing
// "1970-01-01 00:00:00". Fields that are out-of-range are normalized (e.g.,
// October 32 -> November 1) so that all civil-time objects represent valid
// values.
//
// Each civil-time class is aligned to the civil-time field indicated in the
// class's name after normalization. Alignment is performed by setting all the
// inferior fields to their minimum valid value (as described above). The
// following are examples of how each of the six types would align the fields
// representing November 22, 2015 at 12:34:56 in the afternoon. (Note: the
// string format used here is not important; it's just a shorthand way of
// showing the six YMDHMS fields.)
//
// civil_second 2015-11-22 12:34:56
// civil_minute 2015-11-22 12:34:00
// civil_hour 2015-11-22 12:00:00
// civil_day 2015-11-22 00:00:00
// civil_month 2015-11-01 00:00:00
// civil_year 2015-01-01 00:00:00
//
// Each civil-time type performs arithmetic on the field to which it is
// aligned. This means that adding 1 to a civil_day increments the day field
// (normalizing as necessary), and subtracting 7 from a civil_month operates
// on the month field (normalizing as necessary). All arithmetic produces a
// valid civil time. Difference requires two similarly aligned civil-time
// objects and returns the scalar answer in units of the objects' alignment.
// For example, the difference between two civil_hour objects will give an
// answer in units of civil hours.
//
// In addition to the six civil-time types just described, there are
// a handful of helper functions and algorithms for performing common
// calculations. These are described below.
//
// Note: In C++14 and later, this library is usable in a constexpr context.
//
// CONSTRUCTION:
//
// Each of the civil-time types can be constructed in two ways: by directly
// passing to the constructor up to six (optional) integers representing the
// YMDHMS fields, or by copying the YMDHMS fields from a differently aligned
// civil-time type.
//
// civil_day default_value; // 1970-01-01 00:00:00
//
// civil_day a(2015, 2, 3); // 2015-02-03 00:00:00
// civil_day b(2015, 2, 3, 4, 5, 6); // 2015-02-03 00:00:00
// civil_day c(2015); // 2015-01-01 00:00:00
//
// civil_second ss(2015, 2, 3, 4, 5, 6); // 2015-02-03 04:05:06
// civil_minute mm(ss); // 2015-02-03 04:05:00
// civil_hour hh(mm); // 2015-02-03 04:00:00
// civil_day d(hh); // 2015-02-03 00:00:00
// civil_month m(d); // 2015-02-01 00:00:00
// civil_year y(m); // 2015-01-01 00:00:00
//
// m = civil_month(y); // 2015-01-01 00:00:00
// d = civil_day(m); // 2015-01-01 00:00:00
// hh = civil_hour(d); // 2015-01-01 00:00:00
// mm = civil_minute(hh); // 2015-01-01 00:00:00
// ss = civil_second(mm); // 2015-01-01 00:00:00
//
// ALIGNMENT CONVERSION:
//
// The alignment of a civil-time object cannot change, but the object may be
// used to construct a new object with a different alignment. This is referred
// to as "realigning". When realigning to a type with the same or more
// precision (e.g., civil_day -> civil_second), the conversion may be
// performed implicitly since no information is lost. However, if information
// could be discarded (e.g., civil_second -> civil_day), the conversion must
// be explicit at the call site.
//
// void fun(const civil_day& day);
//
// civil_second cs;
// fun(cs); // Won't compile because data may be discarded
// fun(civil_day(cs)); // OK: explicit conversion
//
// civil_day cd;
// fun(cd); // OK: no conversion needed
//
// civil_month cm;
// fun(cm); // OK: implicit conversion to civil_day
//
// NORMALIZATION:
//
// Integer arguments passed to the constructor may be out-of-range, in which
// case they are normalized to produce a valid civil-time object. This enables
// natural arithmetic on constructor arguments without worrying about the
// field's range. Normalization guarantees that there are no invalid
// civil-time objects.
//
// civil_day d(2016, 10, 32); // Out-of-range day; normalized to 2016-11-01
//
// Note: If normalization is undesired, you can signal an error by comparing
// the constructor arguments to the normalized values returned by the YMDHMS
// properties.
//
// PROPERTIES:
//
// All civil-time types have accessors for all six of the civil-time fields:
// year, month, day, hour, minute, and second. Recall that fields inferior to
// the type's aligment will be set to their minimum valid value.
//
// civil_day d(2015, 6, 28);
// // d.year() == 2015
// // d.month() == 6
// // d.day() == 28
// // d.hour() == 0
// // d.minute() == 0
// // d.second() == 0
//
// COMPARISON:
//
// Comparison always considers all six YMDHMS fields, regardless of the type's
// alignment. Comparison between differently aligned civil-time types is
// allowed.
//
// civil_day feb_3(2015, 2, 3); // 2015-02-03 00:00:00
// civil_day mar_4(2015, 3, 4); // 2015-03-04 00:00:00
// // feb_3 < mar_4
// // civil_year(feb_3) == civil_year(mar_4)
//
// civil_second feb_3_noon(2015, 2, 3, 12, 0, 0); // 2015-02-03 12:00:00
// // feb_3 < feb_3_noon
// // feb_3 == civil_day(feb_3_noon)
//
// // Iterates all the days of February 2015.
// for (civil_day d(2015, 2, 1); d < civil_month(2015, 3); ++d) {
// // ...
// }
//
// STREAMING:
//
// Each civil-time type may be sent to an output stream using operator<<().
// The output format follows the pattern "YYYY-MM-DDThh:mm:ss" where fields
// inferior to the type's alignment are omitted.
//
// civil_second cs(2015, 2, 3, 4, 5, 6);
// std::cout << cs << "\n"; // Outputs: 2015-02-03T04:05:06
//
// civil_day cd(cs);
// std::cout << cd << "\n"; // Outputs: 2015-02-03
//
// civil_year cy(cs);
// std::cout << cy << "\n"; // Outputs: 2015
//
// ARITHMETIC:
//
// Civil-time types support natural arithmetic operators such as addition,
// subtraction, and difference. Arithmetic operates on the civil-time field
// indicated in the type's name. Difference requires arguments with the same
// alignment and returns the answer in units of the alignment.
//
// civil_day a(2015, 2, 3);
// ++a; // 2015-02-04 00:00:00
// --a; // 2015-02-03 00:00:00
// civil_day b = a + 1; // 2015-02-04 00:00:00
// civil_day c = 1 + b; // 2015-02-05 00:00:00
// int n = c - a; // n = 2 (civil days)
// int m = c - civil_month(c); // Won't compile: different types.
//
// EXAMPLE: Adding a month to January 31.
//
// One of the classic questions that arises when considering a civil-time
// library (or a date library or a date/time library) is this: "What happens
// when you add a month to January 31?" This is an interesting question
// because there could be a number of possible answers:
//
// 1. March 3 (or 2 if a leap year). This may make sense if the operation
// wants the equivalent of February 31.
// 2. February 28 (or 29 if a leap year). This may make sense if the operation
// wants the last day of January to go to the last day of February.
// 3. Error. The caller may get some error, an exception, an invalid date
// object, or maybe false is returned. This may make sense because there is
// no single unambiguously correct answer to the question.
//
// Practically speaking, any answer that is not what the programmer intended
// is the wrong answer.
//
// This civil-time library avoids the problem by making it impossible to ask
// ambiguous questions. All civil-time objects are aligned to a particular
// civil-field boundary (such as aligned to a year, month, day, hour, minute,
// or second), and arithmetic operates on the field to which the object is
// aligned. This means that in order to "add a month" the object must first be
// aligned to a month boundary, which is equivalent to the first day of that
// month.
//
// Of course, there are ways to compute an answer the question at hand using
// this civil-time library, but they require the programmer to be explicit
// about the answer they expect. To illustrate, let's see how to compute all
// three of the above possible answers to the question of "Jan 31 plus 1
// month":
//
// const civil_day d(2015, 1, 31);
//
// // Answer 1:
// // Add 1 to the month field in the constructor, and rely on normalization.
// const auto ans_normalized = civil_day(d.year(), d.month() + 1, d.day());
// // ans_normalized == 2015-03-03 (aka Feb 31)
//
// // Answer 2:
// // Add 1 to month field, capping to the end of next month.
// const auto next_month = civil_month(d) + 1;
// const auto last_day_of_next_month = civil_day(next_month + 1) - 1;
// const auto ans_capped = std::min(ans_normalized, last_day_of_next_month);
// // ans_capped == 2015-02-28
//
// // Answer 3:
// // Signal an error if the normalized answer is not in next month.
// if (civil_month(ans_normalized) != next_month) {
// // error, month overflow
// }
//
using civil_year = detail::civil_year;
using civil_month = detail::civil_month;
using civil_day = detail::civil_day;
using civil_hour = detail::civil_hour;
using civil_minute = detail::civil_minute;
using civil_second = detail::civil_second;
// An enum class with members monday, tuesday, wednesday, thursday, friday,
// saturday, and sunday. These enum values may be sent to an output stream
// using operator<<(). The result is the full weekday name in English with a
// leading capital letter.
//
// weekday wd = weekday::thursday;
// std::cout << wd << "\n"; // Outputs: Thursday
//
using detail::weekday;
// Returns the weekday for the given civil_day.
//
// civil_day a(2015, 8, 13);
// weekday wd = get_weekday(a); // wd == weekday::thursday
//
using detail::get_weekday;
// Returns the civil_day that strictly follows or precedes the given
// civil_day, and that falls on the given weekday.
//
// For example, given:
//
// August 2015
// Su Mo Tu We Th Fr Sa
// 1
// 2 3 4 5 6 7 8
// 9 10 11 12 13 14 15
// 16 17 18 19 20 21 22
// 23 24 25 26 27 28 29
// 30 31
//
// civil_day a(2015, 8, 13); // get_weekday(a) == weekday::thursday
// civil_day b = next_weekday(a, weekday::thursday); // b = 2015-08-20
// civil_day c = prev_weekday(a, weekday::thursday); // c = 2015-08-06
//
// civil_day d = ...
// // Gets the following Thursday if d is not already Thursday
// civil_day thurs1 = prev_weekday(d, weekday::thursday) + 7;
// // Gets the previous Thursday if d is not already Thursday
// civil_day thurs2 = next_weekday(d, weekday::thursday) - 7;
//
using detail::next_weekday;
using detail::prev_weekday;
// Returns the day-of-year for the given civil_day.
//
// civil_day a(2015, 1, 1);
// int yd_jan_1 = get_yearday(a); // yd_jan_1 = 1
// civil_day b(2015, 12, 31);
// int yd_dec_31 = get_yearday(b); // yd_dec_31 = 365
//
using detail::get_yearday;
} // namespace cctz
#endif // CCTZ_CIVIL_TIME_H_

567
contrib/libcctz/include/civil_time_detail.h Normal file
View File

@ -0,0 +1,567 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <iomanip>
#include <limits>
#include <ostream>
#include <sstream>
#include <type_traits>
// Disable constexpr support unless we are using clang in C++14 mode.
#if __clang__ && __cpp_constexpr >= 201304
#define CONSTEXPR_D constexpr // data
#define CONSTEXPR_F constexpr // function
#define CONSTEXPR_M constexpr // member
#define CONSTEXPR_T constexpr // template
#else
#define CONSTEXPR_D const
#define CONSTEXPR_F inline
#define CONSTEXPR_M
#define CONSTEXPR_T
#endif
namespace cctz {
// Support years that at least span the range of 64-bit time_t values.
using year_t = std::int_fast64_t;
// Type alias that indicates an argument is not normalized (e.g., the
// constructor parameters and operands/results of addition/subtraction).
using diff_t = std::int_fast64_t;
namespace detail {
// Type aliases that indicate normalized argument values.
using month_t = std::int_fast8_t; // [1:12]
using day_t = std::int_fast8_t; // [1:31]
using hour_t = std::int_fast8_t; // [0:23]
using minute_t = std::int_fast8_t; // [0:59]
using second_t = std::int_fast8_t; // [0:59]
// Normalized civil-time fields: Y-M-D HH:MM:SS.
struct fields {
CONSTEXPR_M fields(year_t year, month_t month, day_t day,
hour_t hour, minute_t minute, second_t second)
: y(year), m(month), d(day), hh(hour), mm(minute), ss(second) {}
std::int_least64_t y;
std::int_least8_t m;
std::int_least8_t d;
std::int_least8_t hh;
std::int_least8_t mm;
std::int_least8_t ss;
};
struct second_tag {};
struct minute_tag : second_tag {};
struct hour_tag : minute_tag {};
struct day_tag : hour_tag {};
struct month_tag : day_tag {};
struct year_tag : month_tag {};
////////////////////////////////////////////////////////////////////////
// Field normalization (without avoidable overflow).
namespace impl {
CONSTEXPR_F bool is_leap_year(year_t y) noexcept {
return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0);
}
CONSTEXPR_F int year_index(year_t y, month_t m) noexcept {
return (((y + (m > 2)) % 400) + 400) % 400;
}
CONSTEXPR_F int days_per_century(year_t y, month_t m) noexcept {
const int yi = year_index(y, m);
return 36524 + (yi == 0 || yi > 300);
}
CONSTEXPR_F int days_per_4years(year_t y, month_t m) noexcept {
const int yi = year_index(y, m);
return 1460 + (yi == 0 || yi > 300 || (yi - 1) % 100 < 96);
}
CONSTEXPR_F int days_per_year(year_t y, month_t m) noexcept {
return is_leap_year(y + (m > 2)) ? 366 : 365;
}
CONSTEXPR_F int days_per_month(year_t y, month_t m) noexcept {
CONSTEXPR_D signed char k_days_per_month[12] = {
31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 // non leap year
};
return k_days_per_month[m - 1] + (m == 2 && is_leap_year(y));
}
CONSTEXPR_F fields n_day(year_t y, month_t m, diff_t d, diff_t cd,
hour_t hh, minute_t mm, second_t ss) noexcept {
y += (cd / 146097) * 400;
cd %= 146097;
if (cd < 0) {
y -= 400;
cd += 146097;
}
y += (d / 146097) * 400;
d = d % 146097 + cd;
if (d <= 0) {
y -= 400;
d += 146097;
} else if (d > 146097) {
y += 400;
d -= 146097;
}
if (d > 365) {
for (int n = days_per_century(y, m); d > n; n = days_per_century(y, m)) {
d -= n;
y += 100;
}
for (int n = days_per_4years(y, m); d > n; n = days_per_4years(y, m)) {
d -= n;
y += 4;
}
for (int n = days_per_year(y, m); d > n; n = days_per_year(y, m)) {
d -= n;
++y;
}
}
if (d > 28) {
for (int n = days_per_month(y, m); d > n; n = days_per_month(y, m)) {
d -= n;
if (++m > 12) {
++y;
m = 1;
}
}
}
return fields(y, m, d, hh, mm, ss);
}
CONSTEXPR_F fields n_mon(year_t y, diff_t m, diff_t d, diff_t cd,
hour_t hh, minute_t mm, second_t ss) noexcept {
if (m != 12) {
y += m / 12;
m %= 12;
if (m <= 0) {
y -= 1;
m += 12;
}
}
return n_day(y, m, d, cd, hh, mm, ss);
}
CONSTEXPR_F fields n_hour(year_t y, diff_t m, diff_t d, diff_t cd,
diff_t hh, minute_t mm, second_t ss) noexcept {
cd += hh / 24;
hh %= 24;
if (hh < 0) {
cd -= 1;
hh += 24;
}
return n_mon(y, m, d, cd, hh, mm, ss);
}
CONSTEXPR_F fields n_min(year_t y, diff_t m, diff_t d, diff_t hh, diff_t ch,
diff_t mm, second_t ss) noexcept {
ch += mm / 60;
mm %= 60;
if (mm < 0) {
ch -= 1;
mm += 60;
}
return n_hour(y, m, d, hh / 24 + ch / 24, hh % 24 + ch % 24, mm, ss);
}
CONSTEXPR_F fields n_sec(year_t y, diff_t m, diff_t d, diff_t hh, diff_t mm,
diff_t ss) noexcept {
diff_t cm = ss / 60;
ss %= 60;
if (ss < 0) {
cm -= 1;
ss += 60;
}
return n_min(y, m, d, hh, mm / 60 + cm / 60, mm % 60 + cm % 60, ss);
}
} // namespace impl
////////////////////////////////////////////////////////////////////////
// Increments the indicated (normalized) field by "n".
CONSTEXPR_F fields step(second_tag, fields f, diff_t n) noexcept {
return impl::n_sec(f.y, f.m, f.d, f.hh, f.mm + n / 60, f.ss + n % 60);
}
CONSTEXPR_F fields step(minute_tag, fields f, diff_t n) noexcept {
return impl::n_min(f.y, f.m, f.d, f.hh + n / 60, 0, f.mm + n % 60, f.ss);
}
CONSTEXPR_F fields step(hour_tag, fields f, diff_t n) noexcept {
return impl::n_hour(f.y, f.m, f.d + n / 24, 0, f.hh + n % 24, f.mm, f.ss);
}
CONSTEXPR_F fields step(day_tag, fields f, diff_t n) noexcept {
return impl::n_day(f.y, f.m, f.d, n, f.hh, f.mm, f.ss);
}
CONSTEXPR_F fields step(month_tag, fields f, diff_t n) noexcept {
return impl::n_mon(f.y + n / 12, f.m + n % 12, f.d, 0, f.hh, f.mm, f.ss);
}
CONSTEXPR_F fields step(year_tag, fields f, diff_t n) noexcept {
return fields(f.y + n, f.m, f.d, f.hh, f.mm, f.ss);
}
////////////////////////////////////////////////////////////////////////
namespace impl {
// Returns (v * f + a) but avoiding intermediate overflow when possible.
CONSTEXPR_F diff_t scale_add(diff_t v, diff_t f, diff_t a) noexcept {
return (v < 0) ? ((v + 1) * f + a) - f : ((v - 1) * f + a) + f;
}
// Map a (normalized) Y/M/D to the number of days before/after 1970-01-01.
// Probably overflows for years outside [-292277022656:292277026595].
CONSTEXPR_F diff_t ymd_ord(year_t y, month_t m, day_t d) noexcept {
const diff_t eyear = (m <= 2) ? y - 1 : y;
const diff_t era = (eyear >= 0 ? eyear : eyear - 399) / 400;
const diff_t yoe = eyear - era * 400;
const diff_t doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1;
const diff_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
return era * 146097 + doe - 719468;
}
// Returns the difference in days between two normalized Y-M-D tuples.
// ymd_ord() will encounter integer overflow given extreme year values,
// yet the difference between two such extreme values may actually be
// small, so we take a little care to avoid overflow when possible by
// exploiting the 146097-day cycle.
CONSTEXPR_F diff_t day_difference(year_t y1, month_t m1, day_t d1,
year_t y2, month_t m2, day_t d2) noexcept {
const diff_t a_c4_off = y1 % 400;
const diff_t b_c4_off = y2 % 400;
diff_t c4_diff = (y1 - a_c4_off) - (y2 - b_c4_off);
diff_t delta = ymd_ord(a_c4_off, m1, d1) - ymd_ord(b_c4_off, m2, d2);
if (c4_diff > 0 && delta < 0) {
delta += 2 * 146097;
c4_diff -= 2 * 400;
} else if (c4_diff < 0 && delta > 0) {
delta -= 2 * 146097;
c4_diff += 2 * 400;
}
return (c4_diff / 400 * 146097) + delta;
}
} // namespace impl
// Returns the difference between fields structs using the indicated unit.
CONSTEXPR_F diff_t difference(year_tag, fields f1, fields f2) noexcept {
return f1.y - f2.y;
}
CONSTEXPR_F diff_t difference(month_tag, fields f1, fields f2) noexcept {
return impl::scale_add(difference(year_tag{}, f1, f2), 12, (f1.m - f2.m));
}
CONSTEXPR_F diff_t difference(day_tag, fields f1, fields f2) noexcept {
return impl::day_difference(f1.y, f1.m, f1.d, f2.y, f2.m, f2.d);
}
CONSTEXPR_F diff_t difference(hour_tag, fields f1, fields f2) noexcept {
return impl::scale_add(difference(day_tag{}, f1, f2), 24, (f1.hh - f2.hh));
}
CONSTEXPR_F diff_t difference(minute_tag, fields f1, fields f2) noexcept {
return impl::scale_add(difference(hour_tag{}, f1, f2), 60, (f1.mm - f2.mm));
}
CONSTEXPR_F diff_t difference(second_tag, fields f1, fields f2) noexcept {
return impl::scale_add(difference(minute_tag{}, f1, f2), 60, f1.ss - f2.ss);
}
////////////////////////////////////////////////////////////////////////
// Aligns the (normalized) fields struct to the indicated field.
CONSTEXPR_F fields align(second_tag, fields f) noexcept {
return f;
}
CONSTEXPR_F fields align(minute_tag, fields f) noexcept {
return fields{f.y, f.m, f.d, f.hh, f.mm, 0};
}
CONSTEXPR_F fields align(hour_tag, fields f) noexcept {
return fields{f.y, f.m, f.d, f.hh, 0, 0};
}
CONSTEXPR_F fields align(day_tag, fields f) noexcept {
return fields{f.y, f.m, f.d, 0, 0, 0};
}
CONSTEXPR_F fields align(month_tag, fields f) noexcept {
return fields{f.y, f.m, 1, 0, 0, 0};
}
CONSTEXPR_F fields align(year_tag, fields f) noexcept {
return fields{f.y, 1, 1, 0, 0, 0};
}
////////////////////////////////////////////////////////////////////////
template <typename T>
class civil_time {
public:
explicit CONSTEXPR_M civil_time(year_t y, diff_t m = 1, diff_t d = 1,
diff_t hh = 0, diff_t mm = 0,
diff_t ss = 0) noexcept
: civil_time(impl::n_sec(y, m, d, hh, mm, ss)) {}
CONSTEXPR_M civil_time() noexcept : f_{1970, 1, 1, 0, 0, 0} {}
civil_time(const civil_time&) = default;
civil_time& operator=(const civil_time&) = default;
// Conversion between civil times of different alignment. Conversion to
// a more precise alignment is allowed implicitly (e.g., day -> hour),
// but conversion where information is discarded must be explicit
// (e.g., second -> minute).
template <typename U, typename S>
using preserves_data =
typename std::enable_if<std::is_base_of<U, S>::value>::type;
template <typename U>
CONSTEXPR_M civil_time(const civil_time<U>& ct,
preserves_data<T, U>* = nullptr) noexcept
: civil_time(ct.f_) {}
template <typename U>
explicit CONSTEXPR_M civil_time(const civil_time<U>& ct,
preserves_data<U, T>* = nullptr) noexcept
: civil_time(ct.f_) {}
// Factories for the maximum/minimum representable civil_time.
static civil_time max() {
const auto max_year = std::numeric_limits<std::int_least64_t>::max();
return civil_time(max_year, 12, 31, 23, 59, 59);
}
static civil_time min() {
const auto min_year = std::numeric_limits<std::int_least64_t>::min();
return civil_time(min_year, 1, 1, 0, 0, 0);
}
// Field accessors. Note: All but year() return an int.
CONSTEXPR_M year_t year() const noexcept { return f_.y; }
CONSTEXPR_M int month() const noexcept { return f_.m; }
CONSTEXPR_M int day() const noexcept { return f_.d; }
CONSTEXPR_M int hour() const noexcept { return f_.hh; }
CONSTEXPR_M int minute() const noexcept { return f_.mm; }
CONSTEXPR_M int second() const noexcept { return f_.ss; }
// Assigning arithmetic.
CONSTEXPR_M civil_time& operator+=(diff_t n) noexcept {
f_ = step(T{}, f_, n);
return *this;
}
CONSTEXPR_M civil_time& operator-=(diff_t n) noexcept {
if (n != std::numeric_limits<diff_t>::min()) {
f_ = step(T{}, f_, -n);
} else {
f_ = step(T{}, step(T{}, f_, -(n + 1)), 1);
}
return *this;
}
CONSTEXPR_M civil_time& operator++() noexcept {
return *this += 1;
}
CONSTEXPR_M civil_time operator++(int) noexcept {
const civil_time a = *this;
++*this;
return a;
}
CONSTEXPR_M civil_time& operator--() noexcept {
return *this -= 1;
}
CONSTEXPR_M civil_time operator--(int) noexcept {
const civil_time a = *this;
--*this;
return a;
}
// Binary arithmetic operators.
inline friend CONSTEXPR_M civil_time operator+(civil_time a,
diff_t n) noexcept {
return a += n;
}
inline friend CONSTEXPR_M civil_time operator+(diff_t n,
civil_time a) noexcept {
return a += n;
}
inline friend CONSTEXPR_M civil_time operator-(civil_time a,
diff_t n) noexcept {
return a -= n;
}
inline friend CONSTEXPR_M diff_t operator-(const civil_time& lhs,
const civil_time& rhs) noexcept {
return difference(T{}, lhs.f_, rhs.f_);
}
private:
// All instantiations of this template are allowed to call the following
// private constructor and access the private fields member.
template <typename U>
friend class civil_time;
// The designated constructor that all others eventually call.
explicit CONSTEXPR_M civil_time(fields f) noexcept : f_(align(T{}, f)) {}
fields f_;
};
// Disallows difference between differently aligned types.
// auto n = civil_day(...) - civil_hour(...); // would be confusing.
template <typename Tag1, typename Tag2>
CONSTEXPR_F diff_t operator-(civil_time<Tag1>, civil_time<Tag2>) = delete;
using civil_year = civil_time<year_tag>;
using civil_month = civil_time<month_tag>;
using civil_day = civil_time<day_tag>;
using civil_hour = civil_time<hour_tag>;
using civil_minute = civil_time<minute_tag>;
using civil_second = civil_time<second_tag>;
////////////////////////////////////////////////////////////////////////
// Relational operators that work with differently aligned objects.
// Always compares all six fields.
template <typename T1, typename T2>
CONSTEXPR_T bool operator<(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return (lhs.year() < rhs.year() ||
(lhs.year() == rhs.year() &&
(lhs.month() < rhs.month() ||
(lhs.month() == rhs.month() &&
(lhs.day() < rhs.day() ||
(lhs.day() == rhs.day() &&
(lhs.hour() < rhs.hour() ||
(lhs.hour() == rhs.hour() &&
(lhs.minute() < rhs.minute() ||
(lhs.minute() == rhs.minute() &&
(lhs.second() < rhs.second())))))))))));
}
template <typename T1, typename T2>
CONSTEXPR_T bool operator<=(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return !(rhs < lhs);
}
template <typename T1, typename T2>
CONSTEXPR_T bool operator>=(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return !(lhs < rhs);
}
template <typename T1, typename T2>
CONSTEXPR_T bool operator>(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return rhs < lhs;
}
template <typename T1, typename T2>
CONSTEXPR_T bool operator==(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return lhs.year() == rhs.year() && lhs.month() == rhs.month() &&
lhs.day() == rhs.day() && lhs.hour() == rhs.hour() &&
lhs.minute() == rhs.minute() && lhs.second() == rhs.second();
}
template <typename T1, typename T2>
CONSTEXPR_T bool operator!=(const civil_time<T1>& lhs,
const civil_time<T2>& rhs) noexcept {
return !(lhs == rhs);
}
////////////////////////////////////////////////////////////////////////
// Output stream operators output a format matching YYYY-MM-DDThh:mm:ss,
// while omitting fields inferior to the type's alignment. For example,
// civil_day is formatted only as YYYY-MM-DD.
inline std::ostream& operator<<(std::ostream& os, const civil_year& y) {
std::stringstream ss;
ss << y.year(); // No padding.
return os << ss.str();
}
inline std::ostream& operator<<(std::ostream& os, const civil_month& m) {
std::stringstream ss;
ss << civil_year(m) << '-';
ss << std::setfill('0') << std::setw(2) << m.month();
return os << ss.str();
}
inline std::ostream& operator<<(std::ostream& os, const civil_day& d) {
std::stringstream ss;
ss << civil_month(d) << '-';
ss << std::setfill('0') << std::setw(2) << d.day();
return os << ss.str();
}
inline std::ostream& operator<<(std::ostream& os, const civil_hour& h) {
std::stringstream ss;
ss << civil_day(h) << 'T';
ss << std::setfill('0') << std::setw(2) << h.hour();
return os << ss.str();
}
inline std::ostream& operator<<(std::ostream& os, const civil_minute& m) {
std::stringstream ss;
ss << civil_hour(m) << ':';
ss << std::setfill('0') << std::setw(2) << m.minute();
return os << ss.str();
}
inline std::ostream& operator<<(std::ostream& os, const civil_second& s) {
std::stringstream ss;
ss << civil_minute(s) << ':';
ss << std::setfill('0') << std::setw(2) << s.second();
return os << ss.str();
}
////////////////////////////////////////////////////////////////////////
enum class weekday {
monday,
tuesday,
wednesday,
thursday,
friday,
saturday,
sunday,
};
inline std::ostream& operator<<(std::ostream& os, weekday wd) {
switch (wd) {
case weekday::monday:
return os << "Monday";
case weekday::tuesday:
return os << "Tuesday";
case weekday::wednesday:
return os << "Wednesday";
case weekday::thursday:
return os << "Thursday";
case weekday::friday:
return os << "Friday";
case weekday::saturday:
return os << "Saturday";
case weekday::sunday:
return os << "Sunday";
}
}
CONSTEXPR_F weekday get_weekday(const civil_day& cd) noexcept {
CONSTEXPR_D weekday k_weekday_by_thu_off[] = {
weekday::thursday, weekday::friday, weekday::saturday,
weekday::sunday, weekday::monday, weekday::tuesday,
weekday::wednesday,
};
return k_weekday_by_thu_off[((cd - civil_day()) % 7 + 7) % 7];
}
////////////////////////////////////////////////////////////////////////
CONSTEXPR_F civil_day next_weekday(civil_day cd, weekday wd) noexcept {
do { cd += 1; } while (get_weekday(cd) != wd);
return cd;
}
CONSTEXPR_F civil_day prev_weekday(civil_day cd, weekday wd) noexcept {
do { cd -= 1; } while (get_weekday(cd) != wd);
return cd;
}
CONSTEXPR_F int get_yearday(const civil_day& cd) noexcept {
return cd - civil_day(civil_year(cd)) + 1;
}
} // namespace detail
} // namespace cctz
#undef CONSTEXPR_T
#undef CONSTEXPR_M
#undef CONSTEXPR_F
#undef CONSTEXPR_D

297
contrib/libcctz/include/time_zone.h Normal file
View File

@ -0,0 +1,297 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// A library for translating between absolute times (represented by
// std::chrono::time_points of the std::chrono::system_clock) and civil
// times (represented by cctz::civil_second) using the rules defined by
// a time zone (cctz::time_zone).
#ifndef CCTZ_TIME_ZONE_H_
#define CCTZ_TIME_ZONE_H_
#include <chrono>
#include <cstdint>
#include <string>
#include "civil_time.h"
namespace cctz {
// Convenience aliases. Not intended as public API points.
template <typename D>
using time_point = std::chrono::time_point<std::chrono::system_clock, D>;
using sys_seconds = std::chrono::duration<std::int_fast64_t>;
// cctz::time_zone is an opaque, small, value-type class representing a
// geo-political region within which particular rules are used for mapping
// between absolute and civil times. Time zones are named using the TZ
// identifiers from the IANA Time Zone Database, such as "America/Los_Angeles"
// or "Australia/Sydney". Time zones are created from factory functions such
// as load_time_zone(). Note: strings like "PST" and "EDT" are not valid TZ
// identifiers.
//
// Example:
// cctz::time_zone utc = cctz::utc_time_zone();
// cctz::time_zone loc = cctz::local_time_zone();
// cctz::time_zone lax;
// if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... }
//
// See also:
// - http://www.iana.org/time-zones
// - http://en.wikipedia.org/wiki/Zoneinfo
class time_zone {
public:
time_zone() = default; // Equivalent to UTC
time_zone(const time_zone&) = default;
time_zone& operator=(const time_zone&) = default;
std::string name() const;
// An absolute_lookup represents the civil time (cctz::civil_second) within
// this time_zone at the given absolute time (time_point). There are
// additionally a few other fields that may be useful when working with
// older APIs, such as std::tm.
//
// Example:
// const cctz::time_zone tz = ...
// const auto tp = std::chrono::system_clock::now();
// const cctz::time_zone::absolute_lookup al = tz.lookup(tp);
struct absolute_lookup {
civil_second cs;
// Note: The following fields exist for backward compatibility with older
// APIs. Accessing these fields directly is a sign of imprudent logic in
// the calling code. Modern time-related code should only access this data
// indirectly by way of cctz::format().
int offset; // civil seconds east of UTC
bool is_dst; // is offset non-standard?
std::string abbr; // time-zone abbreviation (e.g., "PST")
};
absolute_lookup lookup(const time_point<sys_seconds>& tp) const;
template <typename D>
absolute_lookup lookup(const time_point<D>& tp) const {
return lookup(std::chrono::time_point_cast<sys_seconds>(tp));
}
// A civil_lookup represents the absolute time(s) (time_point) that
// correspond to the given civil time (cctz::civil_second) within this
// time_zone. Usually the given civil time represents a unique instant in
// time, in which case the conversion is unambiguous and correct. However,
// within this time zone, the given civil time may be skipped (e.g., during
// a positive UTC offset shift), or repeated (e.g., during a negative UTC
// offset shift). To account for these possibilities, civil_lookup is richer
// than just a single output time_point.
//
// In all cases the civil_lookup::kind enum will indicate the nature of the
// given civil-time argument, and the pre, trans, and post, members will
// give the absolute time answers using the pre-transition offset, the
// transition point itself, and the post-transition offset, respectively
// (these are all equal if kind == UNIQUE).
//
// Example:
// cctz::time_zone lax;
// if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... }
//
// // A unique civil time.
// auto jan01 = lax.lookup(cctz::civil_second(2011, 1, 1, 0, 0, 0));
// // jan01.kind == cctz::time_zone::civil_lookup::UNIQUE
// // jan01.pre is 2011/01/01 00:00:00 -0800
// // jan01.trans is 2011/01/01 00:00:00 -0800
// // jan01.post is 2011/01/01 00:00:00 -0800
//
// // A Spring DST transition, when there is a gap in civil time.
// auto mar13 = lax.lookup(cctz::civil_second(2011, 3, 13, 2, 15, 0));
// // mar13.kind == cctz::time_zone::civil_lookup::SKIPPED
// // mar13.pre is 2011/03/13 03:15:00 -0700
// // mar13.trans is 2011/03/13 03:00:00 -0700
// // mar13.post is 2011/03/13 01:15:00 -0800
//
// // A Fall DST transition, when civil times are repeated.
// auto nov06 = lax.lookup(cctz::civil_second(2011, 11, 6, 1, 15, 0));
// // nov06.kind == cctz::time_zone::civil_lookup::REPEATED
// // nov06.pre is 2011/11/06 01:15:00 -0700
// // nov06.trans is 2011/11/06 01:00:00 -0800
// // nov06.post is 2011/11/06 01:15:00 -0800
struct civil_lookup {
enum civil_kind {
UNIQUE, // the civil time was singular (pre == trans == post)
SKIPPED, // the civil time did not exist
REPEATED, // the civil time was ambiguous
} kind;
time_point<sys_seconds> pre; // Uses the pre-transition offset
time_point<sys_seconds> trans; // Instant of civil-offset change
time_point<sys_seconds> post; // Uses the post-transition offset
};
civil_lookup lookup(const civil_second& cs) const;
class Impl;
private:
explicit time_zone(const Impl* impl) : impl_(impl) {}
const Impl* impl_ = nullptr;
};
// Relational operators.
bool operator==(time_zone lhs, time_zone rhs);
inline bool operator!=(time_zone lhs, time_zone rhs) { return !(lhs == rhs); }
// Loads the named time zone. May perform I/O on the initial load.
// If the name is invalid, or some other kind of error occurs, returns
// false and "*tz" is set to the UTC time zone.
bool load_time_zone(const std::string& name, time_zone* tz);
// Returns a time_zone representing UTC. Cannot fail.
time_zone utc_time_zone();
// Returns a time zone representing the local time zone. Falls back to UTC.
time_zone local_time_zone();
// Returns the civil time (cctz::civil_second) within the given time zone at
// the given absolute time (time_point). Since the additional fields provided
// by the time_zone::absolute_lookup struct should rarely be needed in modern
// code, this convert() function is simpler and should be preferred.
template <typename D>
inline civil_second convert(const time_point<D>& tp, const time_zone& tz) {
return tz.lookup(tp).cs;
}
// Returns the absolute time (time_point) that corresponds to the given civil
// time within the given time zone. If the civil time is not unique (i.e., if
// it was either repeated or non-existent), then the returned time_point is
// the best estimate that preserves relative order. That is, this function
// guarantees that if cs1 < cs2, then convert(cs1, tz) <= convert(cs2, tz).
inline time_point<sys_seconds> convert(const civil_second& cs,
const time_zone& tz) {
const time_zone::civil_lookup cl = tz.lookup(cs);
if (cl.kind == time_zone::civil_lookup::SKIPPED) return cl.trans;
return cl.pre;
}
namespace detail {
template <typename D>
inline std::pair<time_point<sys_seconds>, D>
split_seconds(const time_point<D>& tp) {
auto sec = std::chrono::time_point_cast<sys_seconds>(tp);
auto sub = tp - sec;
if (sub.count() < 0) {
sec -= sys_seconds(1);
sub += sys_seconds(1);
}
return {sec, std::chrono::duration_cast<D>(sub)};
}
inline std::pair<time_point<sys_seconds>, sys_seconds>
split_seconds(const time_point<sys_seconds>& tp) {
return {tp, sys_seconds(0)};
}
using femtoseconds = std::chrono::duration<std::int_fast64_t, std::femto>;
std::string format(const std::string&, const time_point<sys_seconds>&,
const femtoseconds&, const time_zone&);
bool parse(const std::string&, const std::string&, const time_zone&,
time_point<sys_seconds>*, femtoseconds*);
} // namespace detail
// Formats the given time_point in the given cctz::time_zone according to
// the provided format string. Uses strftime()-like formatting options,
// with the following extensions:
//
// - %Ez - RFC3339-compatible numeric time zone (+hh:mm or -hh:mm)
// - %E#S - Seconds with # digits of fractional precision
// - %E*S - Seconds with full fractional precision (a literal '*')
// - %E#f - Fractional seconds with # digits of precision
// - %E*f - Fractional seconds with full precision (a literal '*')
// - %E4Y - Four-character years (-999 ... -001, 0000, 0001 ... 9999)
//
// Note that %E0S behaves like %S, and %E0f produces no characters. In
// contrast %E*f always produces at least one digit, which may be '0'.
//
// Note that %Y produces as many characters as it takes to fully render the
// year. A year outside of [-999:9999] when formatted with %E4Y will produce
// more than four characters, just like %Y.
//
// Tip: Format strings should include the UTC offset (e.g., %z or %Ez) so that
// the resultng string uniquely identifies an absolute time.
//
// Example:
// cctz::time_zone lax;
// if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... }
// auto tp = cctz::convert(cctz::civil_second(2013, 1, 2, 3, 4, 5), lax);
// std::string f = cctz::format("%H:%M:%S", tp, lax); // "03:04:05"
// f = cctz::format("%H:%M:%E3S", tp, lax); // "03:04:05.000"
template <typename D>
inline std::string format(const std::string& fmt, const time_point<D>& tp,
const time_zone& tz) {
const auto p = detail::split_seconds(tp);
const auto n = std::chrono::duration_cast<detail::femtoseconds>(p.second);
return detail::format(fmt, p.first, n, tz);
}
// Parses an input string according to the provided format string and
// returns the corresponding time_point. Uses strftime()-like formatting
// options, with the same extensions as cctz::format(), but with the
// exceptions that %E#S is interpreted as %E*S, and %E#f as %E*f.
//
// %Y consumes as many numeric characters as it can, so the matching data
// should always be terminated with a non-numeric. %E4Y always consumes
// exactly four characters, including any sign.
//
// Unspecified fields are taken from the default date and time of ...
//
// "1970-01-01 00:00:00.0 +0000"
//
// For example, parsing a string of "15:45" (%H:%M) will return a time_point
// that represents "1970-01-01 15:45:00.0 +0000".
//
// Note that parse() returns time instants, so it makes most sense to parse
// fully-specified date/time strings that include a UTC offset (%z or %Ez).
//
// Note also that parse() only heeds the fields year, month, day, hour,
// minute, (fractional) second, and UTC offset. Other fields, like weekday (%a
// or %A), while parsed for syntactic validity, are ignored in the conversion.
//
// Date and time fields that are out-of-range will be treated as errors rather
// than normalizing them like cctz::civil_second() would do. For example, it
// is an error to parse the date "Oct 32, 2013" because 32 is out of range.
//
// A second of ":60" is normalized to ":00" of the following minute with
// fractional seconds discarded. The following table shows how the given
// seconds and subseconds will be parsed:
//
// "59.x" -> 59.x // exact
// "60.x" -> 00.0 // normalized
// "00.x" -> 00.x // exact
//
// Errors are indicated by returning false.
//
// Example:
// const cctz::time_zone tz = ...
// std::chrono::system_clock::time_point tp;
// if (cctz::parse("%Y-%m-%d", "2015-10-09", tz, &tp)) {
// ...
// }
template <typename D>
inline bool parse(const std::string& fmt, const std::string& input,
const time_zone& tz, time_point<D>* tpp) {
time_point<sys_seconds> sec;
detail::femtoseconds fs;
const bool b = detail::parse(fmt, input, tz, &sec, &fs);
if (b) {
// TODO: Return false if unrepresentable as a time_point<D>.
*tpp = std::chrono::time_point_cast<D>(sec);
*tpp += std::chrono::duration_cast<D>(fs);
}
return b;
}
} // namespace cctz
#endif // CCTZ_TIME_ZONE_H_

260
contrib/libcctz/src/time_tool.cc Normal file
View File

@ -0,0 +1,260 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// A command-line tool for exercising the CCTZ library.
#include <getopt.h>
#include <algorithm>
#include <cctype>
#include <chrono>
#include <cstring>
#include <ctime>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include "civil_time.h"
#include "time_zone.h"
// Pulls in the aliases from cctz for brevity.
template <typename D>
using time_point = cctz::time_point<D>;
using sys_seconds = cctz::sys_seconds;
// parse() specifiers for command-line time arguments.
const char* const kFormats[] = {
"%Y %m %d %H %M %E*S",
"%Y - %m - %d T %H : %M : %E*S",
"%Y - %m - %d %H : %M : %E*S",
"%Y - %m - %d T %H : %M",
"%Y - %m - %d %H : %M",
"%Y - %m - %d",
"%a %b %d %H : %M : %E*S %Z %Y",
"%a %e %b %Y %H : %M : %E*S",
"%a %b %e %Y %H : %M : %E*S",
"%e %b %Y %H : %M : %E*S",
"%b %e %Y %H : %M : %E*S",
"%a %e %b %Y %H : %M",
"%a %b %e %Y %H : %M",
"%e %b %Y %H : %M",
"%b %e %Y %H : %M",
"%a %e %b %Y",
"%a %b %e %Y",
"%e %b %Y",
"%b %e %Y",
nullptr
};
bool ParseTimeSpec(const std::string& args, cctz::time_zone zone,
time_point<sys_seconds>* when) {
for (const char* const* fmt = kFormats; *fmt != NULL; ++fmt) {
const std::string format = std::string(*fmt) + " %Ez";
time_point<sys_seconds> tp;
if (cctz::parse(format, args, zone, &tp)) {
*when = tp;
return true;
}
}
return false;
}
bool ParseBreakdownSpec(const std::string& args, cctz::civil_second* when) {
const cctz::time_zone utc = cctz::utc_time_zone();
for (const char* const* fmt = kFormats; *fmt != NULL; ++fmt) {
time_point<sys_seconds> tp;
if (cctz::parse(*fmt, args, utc, &tp)) {
*when = cctz::convert(tp, utc);
return true;
}
}
return false;
}
// The FormatTime() specifier for output.
const char* const kFormat = "%Y-%m-%d %H:%M:%S %Ez (%Z)";
const char* WeekDayName(cctz::weekday wd) {
switch (wd) {
case cctz::weekday::monday: return "Mon";
case cctz::weekday::tuesday: return "Tue";
case cctz::weekday::wednesday: return "Wed";
case cctz::weekday::thursday: return "Thu";
case cctz::weekday::friday: return "Fri";
case cctz::weekday::saturday: return "Sat";
case cctz::weekday::sunday: return "Sun";
}
return "XXX";
}
std::string FormatTimeInZone(time_point<sys_seconds> when,
cctz::time_zone zone) {
std::ostringstream oss;
oss << std::setw(33) << std::left << cctz::format(kFormat, when, zone);
cctz::time_zone::absolute_lookup al = zone.lookup(when);
cctz::civil_day cd(al.cs);
oss << " [wd=" << WeekDayName(cctz::get_weekday(cd))
<< " yd=" << std::setw(3) << std::setfill('0')
<< std::right << cctz::get_yearday(cd)
<< " dst=" << (al.is_dst ? 'T' : 'F')
<< " off=" << std::showpos << al.offset << std::noshowpos << "]";
return oss.str();
}
void InstantInfo(const std::string& label, time_point<sys_seconds> when,
cctz::time_zone zone) {
const cctz::time_zone utc = cctz::utc_time_zone(); // might == zone
const std::string time_label = "time_t";
const std::string utc_label = "UTC";
const std::string zone_label = "in-tz"; // perhaps zone.name()?
std::size_t width =
2 + std::max(std::max(time_label.size(), utc_label.size()),
zone_label.size());
std::cout << label << " {\n";
std::cout << std::setw(width) << std::right << time_label << ": ";
std::cout << std::setw(10) << format("%s", when, utc);
std::cout << "\n";
std::cout << std::setw(width) << std::right << utc_label << ": ";
std::cout << FormatTimeInZone(when, utc) << "\n";
std::cout << std::setw(width) << std::right << zone_label << ": ";
std::cout << FormatTimeInZone(when, zone) << "\n";
std::cout << "}\n";
}
// Report everything we know about a cctz::civil_second (YMDHMS).
int BreakdownInfo(const cctz::civil_second& cs, cctz::time_zone zone) {
std::cout << "tz: " << zone.name() << "\n";
cctz::time_zone::civil_lookup cl = zone.lookup(cs);
switch (cl.kind) {
case cctz::time_zone::civil_lookup::UNIQUE: {
std::cout << "kind: UNIQUE\n";
InstantInfo("when", cl.pre, zone);
break;
}
case cctz::time_zone::civil_lookup::SKIPPED: {
std::cout << "kind: SKIPPED\n";
InstantInfo("post", cl.post, zone); // might == trans-1
InstantInfo("trans-1", cl.trans - std::chrono::seconds(1), zone);
InstantInfo("trans", cl.trans, zone);
InstantInfo("pre", cl.pre, zone); // might == trans
break;
}
case cctz::time_zone::civil_lookup::REPEATED: {
std::cout << "kind: REPEATED\n";
InstantInfo("pre", cl.pre, zone); // might == trans-1
InstantInfo("trans-1", cl.trans - std::chrono::seconds(1), zone);
InstantInfo("trans", cl.trans, zone);
InstantInfo("post", cl.post, zone); // might == trans
break;
}
}
return 0;
}
// Report everything we know about a time_point<sys_seconds>.
int TimeInfo(time_point<sys_seconds> when, cctz::time_zone zone) {
std::cout << "tz: " << zone.name() << "\n";
std::cout << "kind: UNIQUE\n";
InstantInfo("when", when, zone);
return 0;
}
const char* Basename(const char* p) {
if (const char* b = strrchr(p, '/')) return ++b;
return p;
}
// std::regex doesn't work before gcc 4.9.
bool LooksLikeNegOffset(const char* s) {
if (s[0] == '-' && std::isdigit(s[1]) && std::isdigit(s[2])) {
int i = (s[3] == ':') ? 4 : 3;
if (std::isdigit(s[i]) && std::isdigit(s[i + 1])) {
return s[i + 2] == '\0';
}
}
return false;
}
int main(int argc, char** argv) {
const std::string prog = argv[0] ? Basename(argv[0]) : "time_tool";
// Escape arguments that look like negative offsets so that they
// don't look like flags.
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "--") == 0) break;
if (LooksLikeNegOffset(argv[i])) {
char* buf = new char[strlen(argv[i] + 2)];
buf[0] = ' '; // will later be ignorned
strcpy(buf + 1, argv[i]);
argv[i] = buf;
}
}
// Determine the time zone.
cctz::time_zone zone = cctz::local_time_zone();
for (;;) {
static option opts[] = {
{"tz", required_argument, nullptr, 'z'},
{nullptr, 0, nullptr, 0},
};
int c = getopt_long(argc, argv, "z:", opts, nullptr);
if (c == -1) break;
switch (c) {
case 'z':
if (!cctz::load_time_zone(optarg, &zone)) {
std::cerr << optarg << ": Unrecognized time zone\n";
return 1;
}
break;
default:
std::cerr << "Usage: " << prog << " [--tz=<zone>] [<time-spec>]\n";
return 1;
}
}
// Determine the time point.
time_point<sys_seconds> tp = std::chrono::time_point_cast<sys_seconds>(
std::chrono::system_clock::now());
std::string args;
for (int i = optind; i < argc; ++i) {
if (i != optind) args += " ";
args += argv[i];
}
std::replace(args.begin(), args.end(), ',', ' ');
std::replace(args.begin(), args.end(), '/', '-');
bool have_time = ParseTimeSpec(args, zone, &tp);
if (!have_time && !args.empty()) {
std::string spec = args.substr((args[0] == '@') ? 1 : 0);
if ((spec.size() > 0 && std::isdigit(spec[0])) ||
(spec.size() > 1 && spec[0] == '-' && std::isdigit(spec[1]))) {
std::size_t end;
const time_t t = std::stoll(spec, &end);
if (end == spec.size()) {
tp = std::chrono::time_point_cast<cctz::sys_seconds>(
std::chrono::system_clock::from_time_t(0)) +
sys_seconds(t);
have_time = true;
}
}
}
cctz::civil_second when = cctz::convert(tp, zone);
bool have_break_down = !have_time && ParseBreakdownSpec(args, &when);
// Show results.
if (have_break_down) return BreakdownInfo(when, zone);
if (have_time || args.empty()) return TimeInfo(tp, zone);
std::cerr << args << ": Malformed time spec\n";
return 1;
}

773
contrib/libcctz/src/time_zone_format.cc Normal file
View File

@ -0,0 +1,773 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if !defined(HAS_STRPTIME)
# if !defined(_MSC_VER)
# define HAS_STRPTIME 1 // assume everyone has strptime() except windows
# endif
#endif
#include "time_zone.h"
#include "time_zone_if.h"
#include <cctype>
#include <chrono>
#include <cstdint>
#include <cstring>
#include <ctime>
#include <limits>
#include <vector>
#if !HAS_STRPTIME
#include <iomanip>
#include <sstream>
#endif
namespace cctz {
namespace detail {
namespace {
#if !HAS_STRPTIME
// Build a strptime() using C++11's std::get_time().
char* strptime(const char* s, const char* fmt, std::tm* tm) {
std::istringstream input(s);
input >> std::get_time(tm, fmt);
if (input.fail()) return nullptr;
return const_cast<char*>(s) + input.tellg();
}
#endif
std::tm ToTM(const time_zone::absolute_lookup& al) {
std::tm tm{};
tm.tm_sec = al.cs.second();
tm.tm_min = al.cs.minute();
tm.tm_hour = al.cs.hour();
tm.tm_mday = al.cs.day();
tm.tm_mon = al.cs.month() - 1;
// Saturate tm.tm_year is cases of over/underflow.
if (al.cs.year() < std::numeric_limits<int>::min() + 1900) {
tm.tm_year = std::numeric_limits<int>::min();
} else if (al.cs.year() - 1900 > std::numeric_limits<int>::max()) {
tm.tm_year = std::numeric_limits<int>::max();
} else {
tm.tm_year = static_cast<int>(al.cs.year() - 1900);
}
switch (get_weekday(civil_day(al.cs))) {
case weekday::sunday:
tm.tm_wday = 0;
break;
case weekday::monday:
tm.tm_wday = 1;
break;
case weekday::tuesday:
tm.tm_wday = 2;
break;
case weekday::wednesday:
tm.tm_wday = 3;
break;
case weekday::thursday:
tm.tm_wday = 4;
break;
case weekday::friday:
tm.tm_wday = 5;
break;
case weekday::saturday:
tm.tm_wday = 6;
break;
}
tm.tm_yday = get_yearday(civil_day(al.cs)) - 1;
tm.tm_isdst = al.is_dst ? 1 : 0;
return tm;
}
const char kDigits[] = "0123456789";
// Formats a 64-bit integer in the given field width. Note that it is up
// to the caller of Format64() [and Format02d()/FormatOffset()] to ensure
// that there is sufficient space before ep to hold the conversion.
char* Format64(char* ep, int width, std::int_fast64_t v) {
bool neg = false;
if (v < 0) {
--width;
neg = true;
if (v == INT64_MIN) {
// Avoid negating INT64_MIN.
int last_digit = -(v % 10);
v /= 10;
if (last_digit < 0) {
++v;
last_digit += 10;
}
--width;
*--ep = kDigits[last_digit];
}
v = -v;
}
do {
--width;
*--ep = kDigits[v % 10];
} while (v /= 10);
while (--width >= 0) *--ep = '0'; // zero pad
if (neg) *--ep = '-';
return ep;
}
// Formats [0 .. 99] as %02d.
char* Format02d(char* ep, int v) {
*--ep = kDigits[v % 10];
*--ep = kDigits[(v / 10) % 10];
return ep;
}
// Formats a UTC offset, like +00:00.
char* FormatOffset(char* ep, int minutes, char sep) {
char sign = '+';
if (minutes < 0) {
minutes = -minutes;
sign = '-';
}
ep = Format02d(ep, minutes % 60);
if (sep != '\0') *--ep = sep;
ep = Format02d(ep, minutes / 60);
*--ep = sign;
return ep;
}
// Formats a std::tm using strftime(3).
void FormatTM(std::string* out, const std::string& fmt, const std::tm& tm) {
// strftime(3) returns the number of characters placed in the output
// array (which may be 0 characters). It also returns 0 to indicate
// an error, like the array wasn't large enough. To accomodate this,
// the following code grows the buffer size from 2x the format string
// length up to 32x.
for (int i = 2; i != 32; i *= 2) {
std::size_t buf_size = fmt.size() * i;
std::vector<char> buf(buf_size);
if (std::size_t len = strftime(&buf[0], buf_size, fmt.c_str(), &tm)) {
out->append(&buf[0], len);
return;
}
}
}
// Used for %E#S/%E#f specifiers and for data values in parse().
template <typename T>
const char* ParseInt(const char* dp, int width, T min, T max, T* vp) {
if (dp != nullptr) {
const T kmin = std::numeric_limits<T>::min();
bool erange = false;
bool neg = false;
T value = 0;
if (*dp == '-') {
neg = true;
if (width <= 0 || --width != 0) {
++dp;
} else {
dp = nullptr; // width was 1
}
}
if (const char* const bp = dp) {
while (const char* cp = strchr(kDigits, *dp)) {
int d = static_cast<int>(cp - kDigits);
if (d >= 10) break;
if (value < kmin / 10) {
erange = true;
break;
}
value *= 10;
if (value < kmin + d) {
erange = true;
break;
}
value -= d;
dp += 1;
if (width > 0 && --width == 0) break;
}
if (dp != bp && !erange && (neg || value != kmin)) {
if (!neg || value != 0) {
if (!neg) value = -value; // make positive
if (min <= value && value <= max) {
*vp = value;
} else {
dp = nullptr;
}
} else {
dp = nullptr;
}
} else {
dp = nullptr;
}
}
}
return dp;
}
// The number of base-10 digits that can be represented by a signed 64-bit
// integer. That is, 10^kDigits10_64 <= 2^63 - 1 < 10^(kDigits10_64 + 1).
const int kDigits10_64 = 18;
// 10^n for everything that can be represented by a signed 64-bit integer.
const std::int_fast64_t kExp10[kDigits10_64 + 1] = {
1,
10,
100,
1000,
10000,
100000,
1000000,
10000000,
100000000,
1000000000,
10000000000,
100000000000,
1000000000000,
10000000000000,
100000000000000,
1000000000000000,
10000000000000000,
100000000000000000,
1000000000000000000,
};
} // namespace
// Uses strftime(3) to format the given Time. The following extended format
// specifiers are also supported:
//
// - %Ez - RFC3339-compatible numeric timezone (+hh:mm or -hh:mm)
// - %E#S - Seconds with # digits of fractional precision
// - %E*S - Seconds with full fractional precision (a literal '*')
// - %E4Y - Four-character years (-999 ... -001, 0000, 0001 ... 9999)
//
// The standard specifiers from RFC3339_* (%Y, %m, %d, %H, %M, and %S) are
// handled internally for performance reasons. strftime(3) is slow due to
// a POSIX requirement to respect changes to ${TZ}.
//
// The TZ/GNU %s extension is handled internally because strftime() has
// to use mktime() to generate it, and that assumes the local time zone.
//
// We also handle the %z and %Z specifiers to accommodate platforms that do
// not support the tm_gmtoff and tm_zone extensions to std::tm.
//
// Requires that zero() <= fs < seconds(1).
std::string format(const std::string& format, const time_point<sys_seconds>& tp,
const detail::femtoseconds& fs, const time_zone& tz) {
std::string result;
const time_zone::absolute_lookup al = tz.lookup(tp);
const std::tm tm = ToTM(al);
// Scratch buffer for internal conversions.
char buf[3 + kDigits10_64]; // enough for longest conversion
char* const ep = buf + sizeof(buf);
char* bp; // works back from ep
// Maintain three, disjoint subsequences that span format.
// [format.begin() ... pending) : already formatted into result
// [pending ... cur) : formatting pending, but no special cases
// [cur ... format.end()) : unexamined
// Initially, everything is in the unexamined part.
const char* pending = format.c_str(); // NUL terminated
const char* cur = pending;
const char* end = pending + format.length();
while (cur != end) { // while something is unexamined
// Moves cur to the next percent sign.
const char* start = cur;
while (cur != end && *cur != '%') ++cur;
// If the new pending text is all ordinary, copy it out.
if (cur != start && pending == start) {
result.append(pending, cur - pending);
pending = start = cur;
}
// Span the sequential percent signs.
const char* percent = cur;
while (cur != end && *cur == '%') ++cur;
// If the new pending text is all percents, copy out one
// percent for every matched pair, then skip those pairs.
if (cur != start && pending == start) {
std::size_t escaped = (cur - pending) / 2;
result.append(pending, escaped);
pending += escaped * 2;
// Also copy out a single trailing percent.
if (pending != cur && cur == end) {
result.push_back(*pending++);
}
}
// Loop unless we have an unescaped percent.
if (cur == end || (cur - percent) % 2 == 0) continue;
// Simple specifiers that we handle ourselves.
if (strchr("YmdeHMSzZs", *cur)) {
if (cur - 1 != pending) {
FormatTM(&result, std::string(pending, cur - 1), tm);
}
switch (*cur) {
case 'Y':
// This avoids the tm.tm_year overflow problem for %Y, however
// tm.tm_year will still be used by other specifiers like %D.
bp = Format64(ep, 0, al.cs.year());
result.append(bp, ep - bp);
break;
case 'm':
bp = Format02d(ep, al.cs.month());
result.append(bp, ep - bp);
break;
case 'd':
case 'e':
bp = Format02d(ep, al.cs.day());
if (*cur == 'e' && *bp == '0') *bp = ' '; // for Windows
result.append(bp, ep - bp);
break;
case 'H':
bp = Format02d(ep, al.cs.hour());
result.append(bp, ep - bp);
break;
case 'M':
bp = Format02d(ep, al.cs.minute());
result.append(bp, ep - bp);
break;
case 'S':
bp = Format02d(ep, al.cs.second());
result.append(bp, ep - bp);
break;
case 'z':
bp = FormatOffset(ep, al.offset / 60, '\0');
result.append(bp, ep - bp);
break;
case 'Z':
result.append(al.abbr);
break;
case 's':
bp = Format64(ep, 0, ToUnixSeconds(tp));
result.append(bp, ep - bp);
break;
}
pending = ++cur;
continue;
}
// Loop if there is no E modifier.
if (*cur != 'E' || ++cur == end) continue;
// Format our extensions.
if (*cur == 'z') {
// Formats %Ez.
if (cur - 2 != pending) {
FormatTM(&result, std::string(pending, cur - 2), tm);
}
bp = FormatOffset(ep, al.offset / 60, ':');
result.append(bp, ep - bp);
pending = ++cur;
} else if (*cur == '*' && cur + 1 != end &&
(*(cur + 1) == 'S' || *(cur + 1) == 'f')) {
// Formats %E*S or %E*F.
if (cur - 2 != pending) {
FormatTM(&result, std::string(pending, cur - 2), tm);
}
char* cp = ep;
bp = Format64(cp, 15, fs.count());
while (cp != bp && cp[-1] == '0') --cp;
switch (*(cur + 1)) {
case 'S':
if (cp != bp) *--bp = '.';
bp = Format02d(bp, al.cs.second());
break;
case 'f':
if (cp == bp) *--bp = '0';
break;
}
result.append(bp, cp - bp);
pending = cur += 2;
} else if (*cur == '4' && cur + 1 != end && *(cur + 1) == 'Y') {
// Formats %E4Y.
if (cur - 2 != pending) {
FormatTM(&result, std::string(pending, cur - 2), tm);
}
bp = Format64(ep, 4, al.cs.year());
result.append(bp, ep - bp);
pending = cur += 2;
} else if (std::isdigit(*cur)) {
// Possibly found %E#S or %E#f.
int n = 0;
if (const char* np = ParseInt(cur, 0, 0, 1024, &n)) {
if (*np == 'S' || *np == 'f') {
// Formats %E#S or %E#f.
if (cur - 2 != pending) {
FormatTM(&result, std::string(pending, cur - 2), tm);
}
bp = ep;
if (n > 0) {
if (n > kDigits10_64) n = kDigits10_64;
bp = Format64(bp, n, (n > 15) ? fs.count() * kExp10[n - 15]
: fs.count() / kExp10[15 - n]);
if (*np == 'S') *--bp = '.';
}
if (*np == 'S') bp = Format02d(bp, al.cs.second());
result.append(bp, ep - bp);
pending = cur = ++np;
}
}
}
}
// Formats any remaining data.
if (end != pending) {
FormatTM(&result, std::string(pending, end), tm);
}
return result;
}
namespace {
const char* ParseOffset(const char* dp, char sep, int* offset) {
if (dp != nullptr) {
const char sign = *dp++;
if (sign == '+' || sign == '-') {
int hours = 0;
const char* ap = ParseInt(dp, 2, 0, 23, &hours);
if (ap != nullptr && ap - dp == 2) {
dp = ap;
if (sep != '\0' && *ap == sep) ++ap;
int minutes = 0;
const char* bp = ParseInt(ap, 2, 0, 59, &minutes);
if (bp != nullptr && bp - ap == 2) dp = bp;
*offset = (hours * 60 + minutes) * 60;
if (sign == '-') *offset = -*offset;
} else {
dp = nullptr;
}
} else {
dp = nullptr;
}
}
return dp;
}
const char* ParseZone(const char* dp, std::string* zone) {
zone->clear();
if (dp != nullptr) {
while (*dp != '\0' && !std::isspace(*dp)) zone->push_back(*dp++);
if (zone->empty()) dp = nullptr;
}
return dp;
}
const char* ParseSubSeconds(const char* dp, detail::femtoseconds* subseconds) {
if (dp != nullptr) {
std::int_fast64_t v = 0;
std::int_fast64_t exp = 0;
const char* const bp = dp;
while (const char* cp = strchr(kDigits, *dp)) {
int d = static_cast<int>(cp - kDigits);
if (d >= 10) break;
if (exp < 15) {
exp += 1;
v *= 10;
v += d;
}
++dp;
}
if (dp != bp) {
v *= kExp10[15 - exp];
*subseconds = detail::femtoseconds(v);
} else {
dp = nullptr;
}
}
return dp;
}
// Parses a string into a std::tm using strptime(3).
const char* ParseTM(const char* dp, const char* fmt, std::tm* tm) {
if (dp != nullptr) {
dp = strptime(dp, fmt, tm);
}
return dp;
}
} // namespace
// Uses strptime(3) to parse the given input. Supports the same extended
// format specifiers as format(), although %E#S and %E*S are treated
// identically (and similarly for %E#f and %E*f).
//
// The standard specifiers from RFC3339_* (%Y, %m, %d, %H, %M, and %S) are
// handled internally so that we can normally avoid strptime() altogether
// (which is particularly helpful when the native implementation is broken).
//
// The TZ/GNU %s extension is handled internally because strptime() has to
// use localtime_r() to generate it, and that assumes the local time zone.
//
// We also handle the %z specifier to accommodate platforms that do not
// support the tm_gmtoff extension to std::tm. %Z is parsed but ignored.
bool parse(const std::string& format, const std::string& input,
const time_zone& tz, time_point<sys_seconds>* sec,
detail::femtoseconds* fs) {
// The unparsed input.
const char* data = input.c_str(); // NUL terminated
// Skips leading whitespace.
while (std::isspace(*data)) ++data;
const year_t kyearmax = std::numeric_limits<year_t>::max();
const year_t kyearmin = std::numeric_limits<year_t>::min();
// Sets default values for unspecified fields.
bool saw_year = false;
year_t year = 1970;
std::tm tm{};
tm.tm_year = 1970 - 1900;
tm.tm_mon = 1 - 1; // Jan
tm.tm_mday = 1;
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;
tm.tm_wday = 4; // Thu
tm.tm_yday = 0;
tm.tm_isdst = 0;
auto subseconds = detail::femtoseconds::zero();
bool saw_offset = false;
int offset = 0; // No offset from passed tz.
std::string zone = "UTC";
const char* fmt = format.c_str(); // NUL terminated
bool twelve_hour = false;
bool afternoon = false;
bool saw_percent_s = false;
std::int_fast64_t percent_s = 0;
// Steps through format, one specifier at a time.
while (data != nullptr && *fmt != '\0') {
if (std::isspace(*fmt)) {
while (std::isspace(*data)) ++data;
while (std::isspace(*++fmt)) continue;
continue;
}
if (*fmt != '%') {
if (*data == *fmt) {
++data;
++fmt;
} else {
data = nullptr;
}
continue;
}
const char* percent = fmt;
if (*++fmt == '\0') {
data = nullptr;
continue;
}
switch (*fmt++) {
case 'Y':
// Symmetrically with FormatTime(), directly handing %Y avoids the
// tm.tm_year overflow problem. However, tm.tm_year will still be
// used by other specifiers like %D.
data = ParseInt(data, 0, kyearmin, kyearmax, &year);
if (data != nullptr) saw_year = true;
continue;
case 'm':
data = ParseInt(data, 2, 1, 12, &tm.tm_mon);
if (data != nullptr) tm.tm_mon -= 1;
continue;
case 'd':
data = ParseInt(data, 2, 1, 31, &tm.tm_mday);
continue;
case 'H':
data = ParseInt(data, 2, 0, 23, &tm.tm_hour);
twelve_hour = false;
continue;
case 'M':
data = ParseInt(data, 2, 0, 59, &tm.tm_min);
continue;
case 'S':
data = ParseInt(data, 2, 0, 60, &tm.tm_sec);
continue;
case 'I':
case 'l':
case 'r': // probably uses %I
twelve_hour = true;
break;
case 'R': // uses %H
case 'T': // uses %H
case 'c': // probably uses %H
case 'X': // probably uses %H
twelve_hour = false;
break;
case 'z':
data = ParseOffset(data, '\0', &offset);
if (data != nullptr) saw_offset = true;
continue;
case 'Z': // ignored; zone abbreviations are ambiguous
data = ParseZone(data, &zone);
continue;
case 's':
data = ParseInt(data, 0, INT_FAST64_MIN, INT_FAST64_MAX, &percent_s);
if (data != nullptr) saw_percent_s = true;
continue;
case 'E':
if (*fmt == 'z') {
if (data != nullptr && *data == 'Z') { // Zulu
offset = 0;
data += 1;
} else {
data = ParseOffset(data, ':', &offset);
}
if (data != nullptr) saw_offset = true;
fmt += 1;
continue;
}
if (*fmt == '*' && *(fmt + 1) == 'S') {
data = ParseInt(data, 2, 0, 60, &tm.tm_sec);
if (data != nullptr && *data == '.') {
data = ParseSubSeconds(data + 1, &subseconds);
}
fmt += 2;
continue;
}
if (*fmt == '*' && *(fmt + 1) == 'f') {
if (data != nullptr && std::isdigit(*data)) {
data = ParseSubSeconds(data, &subseconds);
}
fmt += 2;
continue;
}
if (*fmt == '4' && *(fmt + 1) == 'Y') {
const char* bp = data;
data = ParseInt(data, 4, year_t{-999}, year_t{9999}, &year);
if (data != nullptr) {
if (data - bp == 4) {
saw_year = true;
} else {
data = nullptr; // stopped too soon
}
}
fmt += 2;
continue;
}
if (std::isdigit(*fmt)) {
int n = 0; // value ignored
if (const char* np = ParseInt(fmt, 0, 0, 1024, &n)) {
if (*np == 'S') {
data = ParseInt(data, 2, 0, 60, &tm.tm_sec);
if (data != nullptr && *data == '.') {
data = ParseSubSeconds(data + 1, &subseconds);
}
fmt = ++np;
continue;
}
if (*np == 'f') {
if (data != nullptr && std::isdigit(*data)) {
data = ParseSubSeconds(data, &subseconds);
}
fmt = ++np;
continue;
}
}
}
if (*fmt == 'c') twelve_hour = false; // probably uses %H
if (*fmt == 'X') twelve_hour = false; // probably uses %H
if (*fmt != '\0') ++fmt;
break;
case 'O':
if (*fmt == 'H') twelve_hour = false;
if (*fmt == 'I') twelve_hour = true;
if (*fmt != '\0') ++fmt;
break;
}
// Parses the current specifier.
const char* orig_data = data;
std::string spec(percent, fmt - percent);
data = ParseTM(data, spec.c_str(), &tm);
// If we successfully parsed %p we need to remember whether the result
// was AM or PM so that we can adjust tm_hour before ConvertDateTime().
// So reparse the input with a known AM hour, and check if it is shifted
// to a PM hour.
if (spec == "%p" && data != nullptr) {
std::string test_input = "1" + std::string(orig_data, data - orig_data);
const char* test_data = test_input.c_str();
std::tm tmp{};
ParseTM(test_data, "%I%p", &tmp);
afternoon = (tmp.tm_hour == 13);
}
}
// Adjust a 12-hour tm_hour value if it should be in the afternoon.
if (twelve_hour && afternoon && tm.tm_hour < 12) {
tm.tm_hour += 12;
}
if (data == nullptr) return false;
// Skip any remaining whitespace.
while (std::isspace(*data)) ++data;
// parse() must consume the entire input string.
if (*data != '\0') return false;
// If we saw %s then we ignore anything else and return that time.
if (saw_percent_s) {
*sec = FromUnixSeconds(percent_s);
*fs = detail::femtoseconds::zero();
return true;
}
// If we saw %z or %Ez then we want to interpret the parsed fields in
// UTC and then shift by that offset. Otherwise we want to interpret
// the fields directly in the passed time_zone.
time_zone ptz = saw_offset ? utc_time_zone() : tz;
// Allows a leap second of 60 to normalize forward to the following ":00".
if (tm.tm_sec == 60) {
tm.tm_sec -= 1;
offset -= 1;
subseconds = detail::femtoseconds::zero();
}
if (!saw_year) {
year = year_t{tm.tm_year};
if (year > kyearmax - 1900) return false;
year += 1900;
}
// TODO: Eliminate extra normalization.
const civil_second cs(year, tm.tm_mon + 1, tm.tm_mday,
tm.tm_hour, tm.tm_min, tm.tm_sec);
// parse() fails if any normalization was done. That is,
// parsing "Sep 31" will not produce the equivalent of "Oct 1".
if (cs.year() != year || cs.month() != tm.tm_mon + 1 ||
cs.day() != tm.tm_mday || cs.hour() != tm.tm_hour ||
cs.minute() != tm.tm_min || cs.second() != tm.tm_sec) {
return false;
}
*sec = ptz.lookup(cs).pre - sys_seconds(offset);
*fs = subseconds;
return true;
}
} // namespace detail
} // namespace cctz

34
contrib/libcctz/src/time_zone_if.cc Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "time_zone_if.h"
#include "time_zone_info.h"
#include "time_zone_libc.h"
namespace cctz {
std::unique_ptr<TimeZoneIf> TimeZoneIf::Load(const std::string& name) {
// Support "libc:localtime" and "libc:*" to access the legacy
// localtime and UTC support respectively from the C library.
if (name.compare(0, 5, "libc:") == 0) {
return std::unique_ptr<TimeZoneIf>(new TimeZoneLibC(name.substr(5)));
}
// Otherwise use the "zoneinfo" implementation by default.
std::unique_ptr<TimeZoneInfo> tz(new TimeZoneInfo);
if (!tz->Load(name)) tz.reset();
return std::unique_ptr<TimeZoneIf>(tz.release());
}
} // namespace cctz

62
contrib/libcctz/src/time_zone_if.h Normal file
View File

@ -0,0 +1,62 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CCTZ_TIME_ZONE_IF_H_
#define CCTZ_TIME_ZONE_IF_H_
#include <cstdint>
#include <memory>
#include <string>
#include "civil_time.h"
#include "time_zone.h"
namespace cctz {
// A simple interface used to hide time-zone complexities from time_zone::Impl.
// Subclasses implement the functions for civil-time conversions in the zone.
class TimeZoneIf {
public:
// A factory function for TimeZoneIf implementations.
static std::unique_ptr<TimeZoneIf> Load(const std::string& name);
virtual ~TimeZoneIf() {}
virtual time_zone::absolute_lookup BreakTime(
const time_point<sys_seconds>& tp) const = 0;
virtual time_zone::civil_lookup MakeTime(
const civil_second& cs) const = 0;
protected:
TimeZoneIf() {}
};
// Converts tp to a count of seconds since the Unix epoch.
inline std::int_fast64_t ToUnixSeconds(const time_point<sys_seconds>& tp) {
return (tp - std::chrono::time_point_cast<sys_seconds>(
std::chrono::system_clock::from_time_t(0)))
.count();
}
// Converts a count of seconds since the Unix epoch to a
// time_point<sys_seconds>.
inline time_point<sys_seconds> FromUnixSeconds(std::int_fast64_t t) {
return std::chrono::time_point_cast<sys_seconds>(
std::chrono::system_clock::from_time_t(0)) +
sys_seconds(t);
}
} // namespace cctz
#endif // CCTZ_TIME_ZONE_IF_H_

99
contrib/libcctz/src/time_zone_impl.cc Normal file
View File

@ -0,0 +1,99 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "time_zone_impl.h"
#include <mutex>
#include <unordered_map>
namespace cctz {
namespace {
// time_zone::Impls are linked into a map to support fast lookup by name.
using TimeZoneImplByName =
std::unordered_map<std::string, const time_zone::Impl*>;
TimeZoneImplByName* time_zone_map = nullptr;
// Mutual exclusion for time_zone_map.
std::mutex time_zone_mutex;
} // namespace
time_zone time_zone::Impl::UTC() {
return time_zone(UTCImpl());
}
bool time_zone::Impl::LoadTimeZone(const std::string& name, time_zone* tz) {
// First check for UTC.
if (name.compare("UTC") == 0) {
*tz = time_zone(UTCImpl());
return true;
}
// Then check, under a shared lock, whether the time zone has already
// been loaded. This is the common path. TODO: Move to shared_mutex.
{
std::lock_guard<std::mutex> lock(time_zone_mutex);
if (time_zone_map != nullptr) {
TimeZoneImplByName::const_iterator itr = time_zone_map->find(name);
if (itr != time_zone_map->end()) {
*tz = time_zone(itr->second);
return itr->second != UTCImpl();
}
}
}
// Now check again, under an exclusive lock.
std::lock_guard<std::mutex> lock(time_zone_mutex);
if (time_zone_map == nullptr) time_zone_map = new TimeZoneImplByName;
const Impl*& impl = (*time_zone_map)[name];
bool fallback_utc = false;
if (impl == nullptr) {
// The first thread in loads the new time zone.
Impl* new_impl = new Impl(name);
new_impl->zone_ = TimeZoneIf::Load(new_impl->name_);
if (new_impl->zone_ == nullptr) {
delete new_impl; // free the nascent Impl
impl = UTCImpl(); // and fallback to UTC
fallback_utc = true;
} else {
impl = new_impl; // install new time zone
}
}
*tz = time_zone(impl);
return !fallback_utc;
}
const time_zone::Impl& time_zone::Impl::get(const time_zone& tz) {
if (tz.impl_ == nullptr) {
// Dereferencing an implicit-UTC time_zone is expected to be
// rare, so we don't mind paying a small synchronization cost.
return *UTCImpl();
}
return *tz.impl_;
}
time_zone::Impl::Impl(const std::string& name) : name_(name) {}
const time_zone::Impl* time_zone::Impl::UTCImpl() {
static Impl* utc_impl = [] {
Impl* impl = new Impl("UTC");
impl->zone_ = TimeZoneIf::Load(impl->name_); // never fails
return impl;
}();
return utc_impl;
}
} // namespace cctz

65
contrib/libcctz/src/time_zone_impl.h Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CCTZ_TIME_ZONE_IMPL_H_
#define CCTZ_TIME_ZONE_IMPL_H_
#include <memory>
#include <string>
#include "time_zone.h"
#include "time_zone_info.h"
namespace cctz {
// time_zone::Impl is the internal object referenced by a cctz::time_zone.
class time_zone::Impl {
public:
// The UTC time zone. Also used for other time zones that fail to load.
static time_zone UTC();
// Load a named time zone. Returns false if the name is invalid, or if
// some other kind of error occurs. Note that loading "UTC" never fails.
static bool LoadTimeZone(const std::string& name, time_zone* tz);
// Dereferences the time_zone to obtain its Impl.
static const time_zone::Impl& get(const time_zone& tz);
// The primary key is the time-zone ID (e.g., "America/New_York").
const std::string& name() const { return name_; }
// Breaks a time_point down to civil-time components in this time zone.
time_zone::absolute_lookup BreakTime(
const time_point<sys_seconds>& tp) const {
return zone_->BreakTime(tp);
}
// Converts the civil-time components in this time zone into a time_point.
// That is, the opposite of BreakTime(). The requested civil time may be
// ambiguous or illegal due to a change of UTC offset.
time_zone::civil_lookup MakeTime(const civil_second& cs) const {
return zone_->MakeTime(cs);
}
private:
explicit Impl(const std::string& name);
static const Impl* UTCImpl();
const std::string name_;
std::unique_ptr<TimeZoneIf> zone_;
};
} // namespace cctz
#endif // CCTZ_TIME_ZONE_IMPL_H_

738
contrib/libcctz/src/time_zone_info.cc Normal file
View File

@ -0,0 +1,738 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file implements the TimeZoneIf interface using the "zoneinfo"
// data provided by the IANA Time Zone Database (i.e., the only real game
// in town).
//
// TimeZoneInfo represents the history of UTC-offset changes within a time
// zone. Most changes are due to daylight-saving rules, but occasionally
// shifts are made to the time-zone's base offset. The database only attempts
// to be definitive for times since 1970, so be wary of local-time conversions
// before that. Also, rule and zone-boundary changes are made at the whim
// of governments, so the conversion of future times needs to be taken with
// a grain of salt.
//
// For more information see tzfile(5), http://www.iana.org/time-zones, or
// http://en.wikipedia.org/wiki/Zoneinfo.
//
// Note that we assume the proleptic Gregorian calendar and 60-second
// minutes throughout.
#include "time_zone_info.h"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <limits>
#include "time_zone_posix.h"
namespace cctz {
namespace {
// Convert errnum to a message, using buf[buflen] if necessary.
// buf must be non-null, and buflen non-zero.
char* errmsg(int errnum, char* buf, std::size_t buflen) {
#if defined(_MSC_VER)
strerror_s(buf, buflen, errnum);
return buf;
#elif defined(__APPLE__)
strerror_r(errnum, buf, buflen);
return buf;
#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE
strerror_r(errnum, buf, buflen);
return buf;
#else
return strerror_r(errnum, buf, buflen);
#endif
}
// Wrap the tzfile.h isleap() macro with an inline function, which will
// then have normal argument-passing semantics (i.e., single evaluation).
inline bool IsLeap(cctz::year_t year) { return isleap(year); }
// The day offsets of the beginning of each (1-based) month in non-leap
// and leap years respectively. That is, sigma[1:n]:kDaysPerMonth[][i].
// For example, in a leap year there are 335 days before December.
const std::int_least16_t kMonthOffsets[2][1 + MONSPERYEAR + 1] = {
{-1, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365},
{-1, 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366},
};
// 400-year chunks always have 146097 days (20871 weeks).
const std::int_least64_t kSecPer400Years = 146097LL * SECSPERDAY;
// The number of seconds in non-leap and leap years respectively.
const std::int_least32_t kSecPerYear[2] = {
DAYSPERNYEAR * SECSPERDAY,
DAYSPERLYEAR * SECSPERDAY,
};
// Like kSecPerYear[] but scaled down by a factor of SECSPERDAY.
const std::int_least32_t kDaysPerYear[2] = {DAYSPERNYEAR, DAYSPERLYEAR};
// January 1st at 00:00:00 in the epoch year.
const civil_second unix_epoch(EPOCH_YEAR, 1, 1, 0, 0, 0);
// Single-byte, unsigned numeric values are encoded directly.
inline std::uint_fast8_t Decode8(const char* cp) {
return static_cast<std::uint_fast8_t>(*cp) & 0xff;
}
// Multi-byte, numeric values are encoded using a MSB first,
// twos-complement representation. These helpers decode, from
// the given address, 4-byte and 8-byte values respectively.
std::int_fast32_t Decode32(const char* cp) {
std::uint_fast32_t v = 0;
for (int i = 0; i != (32 / 8); ++i) v = (v << 8) | Decode8(cp++);
if (v <= INT32_MAX) return static_cast<std::int_fast32_t>(v);
return static_cast<std::int_fast32_t>(v - INT32_MAX - 1) + INT32_MIN;
}
std::int_fast64_t Decode64(const char* cp) {
std::uint_fast64_t v = 0;
for (int i = 0; i != (64 / 8); ++i) v = (v << 8) | Decode8(cp++);
if (v <= INT64_MAX) return static_cast<std::int_fast64_t>(v);
return static_cast<std::int_fast64_t>(v - INT64_MAX - 1) + INT64_MIN;
}
// Generate a year-relative offset for a PosixTransition.
std::int_fast64_t TransOffset(bool leap_year, int jan1_weekday,
const PosixTransition& pt) {
std::int_fast64_t days = 0;
switch (pt.date.fmt) {
case PosixTransition::J: {
days = pt.date.j.day;
if (!leap_year || days < kMonthOffsets[1][TM_MARCH + 1]) days -= 1;
break;
}
case PosixTransition::N: {
days = pt.date.n.day;
break;
}
case PosixTransition::M: {
const bool last_week = (pt.date.m.week == 5);
days = kMonthOffsets[leap_year][pt.date.m.month + last_week];
const int weekday = (jan1_weekday + days) % DAYSPERWEEK;
if (last_week) {
days -=
(weekday + DAYSPERWEEK - 1 - pt.date.m.weekday) % DAYSPERWEEK + 1;
} else {
days += (pt.date.m.weekday + DAYSPERWEEK - weekday) % DAYSPERWEEK;
days += (pt.date.m.week - 1) * DAYSPERWEEK;
}
break;
}
}
return (days * SECSPERDAY) + pt.time.offset;
}
inline time_zone::civil_lookup MakeUnique(std::int_fast64_t unix_time) {
time_zone::civil_lookup cl;
cl.pre = cl.trans = cl.post = FromUnixSeconds(unix_time);
cl.kind = time_zone::civil_lookup::UNIQUE;
return cl;
}
inline time_zone::civil_lookup MakeSkipped(const Transition& tr,
const DateTime& dt) {
time_zone::civil_lookup cl;
cl.pre = FromUnixSeconds(tr.unix_time - 1 + (dt - tr.prev_date_time));
cl.trans = FromUnixSeconds(tr.unix_time);
cl.post = FromUnixSeconds(tr.unix_time - (tr.date_time - dt));
cl.kind = time_zone::civil_lookup::SKIPPED;
return cl;
}
inline time_zone::civil_lookup MakeRepeated(const Transition& tr,
const DateTime& dt) {
time_zone::civil_lookup cl;
cl.pre = FromUnixSeconds(tr.unix_time - 1 - (tr.prev_date_time - dt));
cl.trans = FromUnixSeconds(tr.unix_time);
cl.post = FromUnixSeconds(tr.unix_time + (dt - tr.date_time));
cl.kind = time_zone::civil_lookup::REPEATED;
return cl;
}
civil_second YearShift(const civil_second& cs, cctz::year_t year_shift) {
// TODO: How do we do this while avoiding any normalization tests?
return civil_second(cs.year() + year_shift, cs.month(), cs.day(),
cs.hour(), cs.minute(), cs.second());
}
} // namespace
// Assign from a civil_second, created using a TimeZoneInfo timestamp.
void DateTime::Assign(const civil_second& cs) {
offset = cs - unix_epoch;
}
// What (no leap-seconds) UTC+seconds zoneinfo would look like.
bool TimeZoneInfo::ResetToBuiltinUTC(std::int_fast32_t seconds) {
transition_types_.resize(1);
TransitionType& tt(transition_types_.back());
tt.utc_offset = seconds;
tt.is_dst = false;
tt.abbr_index = 0;
transitions_.clear();
transitions_.reserve(2);
for (const std::int_fast64_t unix_time : {-(1LL << 59), 2147483647LL}) {
Transition& tr(*transitions_.emplace(transitions_.end()));
tr.unix_time = unix_time;
tr.type_index = 0;
tr.date_time.Assign(LocalTime(tr.unix_time, tt).cs);
tr.prev_date_time = tr.date_time;
tr.prev_date_time.offset -= 1;
}
default_transition_type_ = 0;
abbreviations_ = "UTC"; // TODO: Handle non-zero offset.
abbreviations_.append(1, '\0'); // add NUL
future_spec_.clear(); // never needed for a fixed-offset zone
extended_ = false;
transitions_.shrink_to_fit();
return true;
}
// Builds the in-memory header using the raw bytes from the file.
void TimeZoneInfo::Header::Build(const tzhead& tzh) {
timecnt = Decode32(tzh.tzh_timecnt);
typecnt = Decode32(tzh.tzh_typecnt);
charcnt = Decode32(tzh.tzh_charcnt);
leapcnt = Decode32(tzh.tzh_leapcnt);
ttisstdcnt = Decode32(tzh.tzh_ttisstdcnt);
ttisgmtcnt = Decode32(tzh.tzh_ttisgmtcnt);
}
// How many bytes of data are associated with this header. The result
// depends upon whether this is a section with 4-byte or 8-byte times.
std::size_t TimeZoneInfo::Header::DataLength(std::size_t time_len) const {
std::size_t len = 0;
len += (time_len + 1) * timecnt; // unix_time + type_index
len += (4 + 1 + 1) * typecnt; // utc_offset + is_dst + abbr_index
len += 1 * charcnt; // abbreviations
len += (time_len + 4) * leapcnt; // leap-time + TAI-UTC
len += 1 * ttisstdcnt; // UTC/local indicators
len += 1 * ttisgmtcnt; // standard/wall indicators
return len;
}
// Check that the TransitionType has the expected offset/is_dst/abbreviation.
void TimeZoneInfo::CheckTransition(const std::string& name,
const TransitionType& tt,
std::int_fast32_t offset, bool is_dst,
const std::string& abbr) const {
if (tt.utc_offset != offset || tt.is_dst != is_dst ||
&abbreviations_[tt.abbr_index] != abbr) {
std::clog << name << ": Transition"
<< " offset=" << tt.utc_offset << "/"
<< (tt.is_dst ? "DST" : "STD")
<< "/abbr=" << &abbreviations_[tt.abbr_index]
<< " does not match POSIX spec '" << future_spec_ << "'\n";
}
}
// zic(8) can generate no-op transitions when a zone changes rules at an
// instant when there is actually no discontinuity. So we check whether
// two transitions have equivalent types (same offset/is_dst/abbr).
bool TimeZoneInfo::EquivTransitions(std::uint_fast8_t tt1_index,
std::uint_fast8_t tt2_index) const {
if (tt1_index == tt2_index) return true;
const TransitionType& tt1(transition_types_[tt1_index]);
const TransitionType& tt2(transition_types_[tt2_index]);
if (tt1.is_dst != tt2.is_dst) return false;
if (tt1.utc_offset != tt2.utc_offset) return false;
if (tt1.abbr_index != tt2.abbr_index) return false;
return true;
}
// Use the POSIX-TZ-environment-variable-style string to handle times
// in years after the last transition stored in the zoneinfo data.
void TimeZoneInfo::ExtendTransitions(const std::string& name,
const Header& hdr) {
extended_ = false;
bool extending = !future_spec_.empty();
PosixTimeZone posix;
if (extending && !ParsePosixSpec(future_spec_, &posix)) {
std::clog << name << ": Failed to parse '" << future_spec_ << "'\n";
extending = false;
}
if (extending && posix.dst_abbr.empty()) { // std only
// The future specification should match the last/default transition,
// and that means that handling the future will fall out naturally.
std::uint_fast8_t index = default_transition_type_;
if (hdr.timecnt != 0) index = transitions_[hdr.timecnt - 1].type_index;
const TransitionType& tt(transition_types_[index]);
CheckTransition(name, tt, posix.std_offset, false, posix.std_abbr);
extending = false;
}
if (extending && hdr.timecnt < 2) {
std::clog << name << ": Too few transitions for POSIX spec\n";
extending = false;
}
if (!extending) {
// Ensure that there is always a transition in the second half of the
// time line (the BIG_BANG transition is in the first half) so that the
// signed difference between a DateTime and the DateTime of its previous
// transition is always representable, without overflow.
const Transition& last(transitions_.back());
if (last.unix_time < 0) {
const std::uint_fast8_t type_index = last.type_index;
Transition& tr(*transitions_.emplace(transitions_.end()));
tr.unix_time = 2147483647; // 2038-01-19T03:14:07+00:00
tr.type_index = type_index;
}
return; // last transition wins
}
// Extend the transitions for an additional 400 years using the
// future specification. Years beyond those can be handled by
// mapping back to a cycle-equivalent year within that range.
// zic(8) should probably do this so that we don't have to.
transitions_.resize(hdr.timecnt + 400 * 2);
extended_ = true;
// The future specification should match the last two transitions,
// and those transitions should have different is_dst flags but be
// in the same year.
// TODO: Investigate the actual guarantees made by zic.
const Transition& tr0(transitions_[hdr.timecnt - 1]);
const Transition& tr1(transitions_[hdr.timecnt - 2]);
const TransitionType& tt0(transition_types_[tr0.type_index]);
const TransitionType& tt1(transition_types_[tr1.type_index]);
const TransitionType& spring(tt0.is_dst ? tt0 : tt1);
const TransitionType& autumn(tt0.is_dst ? tt1 : tt0);
CheckTransition(name, spring, posix.dst_offset, true, posix.dst_abbr);
CheckTransition(name, autumn, posix.std_offset, false, posix.std_abbr);
last_year_ = LocalTime(tr0.unix_time, tt0).cs.year();
if (LocalTime(tr1.unix_time, tt1).cs.year() != last_year_) {
std::clog << name << ": Final transitions not in same year\n";
}
// Add the transitions to tr1 and back to tr0 for each extra year.
const PosixTransition& pt1(tt0.is_dst ? posix.dst_end : posix.dst_start);
const PosixTransition& pt0(tt0.is_dst ? posix.dst_start : posix.dst_end);
Transition* tr = &transitions_[hdr.timecnt]; // next trans to fill
const civil_day jan1(last_year_, 1, 1);
std::int_fast64_t jan1_time = civil_second(jan1) - unix_epoch;
int jan1_weekday = (static_cast<int>(get_weekday(jan1)) + 1) % DAYSPERWEEK;
bool leap_year = IsLeap(last_year_);
for (const cctz::year_t limit = last_year_ + 400; last_year_ < limit;) {
last_year_ += 1; // an additional year of generated transitions
jan1_time += kSecPerYear[leap_year];
jan1_weekday = (jan1_weekday + kDaysPerYear[leap_year]) % DAYSPERWEEK;
leap_year = !leap_year && IsLeap(last_year_);
tr->unix_time =
jan1_time + TransOffset(leap_year, jan1_weekday, pt1) - tt0.utc_offset;
tr++->type_index = tr1.type_index;
tr->unix_time =
jan1_time + TransOffset(leap_year, jan1_weekday, pt0) - tt1.utc_offset;
tr++->type_index = tr0.type_index;
}
}
bool TimeZoneInfo::Load(const std::string& name, FILE* fp) {
// Read and validate the header.
tzhead tzh;
if (fread(&tzh, 1, sizeof tzh, fp) != sizeof tzh)
return false;
if (strncmp(tzh.tzh_magic, TZ_MAGIC, sizeof(tzh.tzh_magic)) != 0)
return false;
Header hdr;
hdr.Build(tzh);
std::size_t time_len = 4;
if (tzh.tzh_version[0] != '\0') {
// Skip the 4-byte data.
if (fseek(fp, static_cast<long>(hdr.DataLength(time_len)), SEEK_CUR) != 0)
return false;
// Read and validate the header for the 8-byte data.
if (fread(&tzh, 1, sizeof tzh, fp) != sizeof tzh)
return false;
if (strncmp(tzh.tzh_magic, TZ_MAGIC, sizeof(tzh.tzh_magic)) != 0)
return false;
if (tzh.tzh_version[0] == '\0')
return false;
hdr.Build(tzh);
time_len = 8;
}
if (hdr.timecnt < 0 || hdr.typecnt <= 0)
return false;
if (hdr.leapcnt != 0) {
// This code assumes 60-second minutes so we do not want
// the leap-second encoded zoneinfo. We could reverse the
// compensation, but it's never in a Google zoneinfo anyway,
// so currently we simply reject such data.
return false;
}
if (hdr.ttisstdcnt != 0 && hdr.ttisstdcnt != hdr.typecnt)
return false;
if (hdr.ttisgmtcnt != 0 && hdr.ttisgmtcnt != hdr.typecnt)
return false;
// Read the data into a local buffer.
std::size_t len = hdr.DataLength(time_len);
std::vector<char> tbuf(len);
if (fread(tbuf.data(), 1, len, fp) != len)
return false;
const char* bp = tbuf.data();
// Decode and validate the transitions.
transitions_.reserve(hdr.timecnt + 2); // We might add a couple.
transitions_.resize(hdr.timecnt);
for (std::int_fast32_t i = 0; i != hdr.timecnt; ++i) {
transitions_[i].unix_time = (time_len == 4) ? Decode32(bp) : Decode64(bp);
bp += time_len;
if (i != 0) {
// Check that the transitions are ordered by time (as zic guarantees).
if (!Transition::ByUnixTime()(transitions_[i - 1], transitions_[i]))
return false; // out of order
}
}
bool seen_type_0 = false;
for (std::int_fast32_t i = 0; i != hdr.timecnt; ++i) {
transitions_[i].type_index = Decode8(bp++);
if (transitions_[i].type_index >= hdr.typecnt)
return false;
if (transitions_[i].type_index == 0)
seen_type_0 = true;
}
// Decode and validate the transition types.
transition_types_.resize(hdr.typecnt);
for (std::int_fast32_t i = 0; i != hdr.typecnt; ++i) {
transition_types_[i].utc_offset = Decode32(bp);
if (transition_types_[i].utc_offset >= SECSPERDAY ||
transition_types_[i].utc_offset <= -SECSPERDAY)
return false;
bp += 4;
transition_types_[i].is_dst = (Decode8(bp++) != 0);
transition_types_[i].abbr_index = Decode8(bp++);
if (transition_types_[i].abbr_index >= hdr.charcnt)
return false;
}
// Determine the before-first-transition type.
default_transition_type_ = 0;
if (seen_type_0 && hdr.timecnt != 0) {
std::uint_fast8_t index = 0;
if (transition_types_[0].is_dst) {
index = transitions_[0].type_index;
while (index != 0 && transition_types_[index].is_dst)
--index;
}
while (index != hdr.typecnt && transition_types_[index].is_dst)
++index;
if (index != hdr.typecnt)
default_transition_type_ = index;
}
// Copy all the abbreviations.
abbreviations_.assign(bp, hdr.charcnt);
bp += hdr.charcnt;
// Skip the unused portions. We've already dispensed with leap-second
// encoded zoneinfo. The ttisstd/ttisgmt indicators only apply when
// interpreting a POSIX spec that does not include start/end rules, and
// that isn't the case here (see "zic -p").
bp += (8 + 4) * hdr.leapcnt; // leap-time + TAI-UTC
bp += 1 * hdr.ttisstdcnt; // UTC/local indicators
bp += 1 * hdr.ttisgmtcnt; // standard/wall indicators
future_spec_.clear();
if (tzh.tzh_version[0] != '\0') {
// Snarf up the NL-enclosed future POSIX spec. Note
// that version '3' files utilize an extended format.
if (fgetc(fp) != '\n')
return false;
for (int c = fgetc(fp); c != '\n'; c = fgetc(fp)) {
if (c == EOF)
return false;
future_spec_.push_back(static_cast<char>(c));
}
}
// We don't check for EOF so that we're forwards compatible.
// Trim redundant transitions. zic may have added these to work around
// differences between the glibc and reference implementations (see
// zic.c:dontmerge) and the Qt library (see zic.c:WORK_AROUND_QTBUG_53071).
// For us, they just get in the way when we do future_spec_ extension.
while (hdr.timecnt > 1) {
if (!EquivTransitions(transitions_[hdr.timecnt - 1].type_index,
transitions_[hdr.timecnt - 2].type_index)) {
break;
}
hdr.timecnt -= 1;
}
transitions_.resize(hdr.timecnt);
// Ensure that there is always a transition in the first half of the
// time line (the second half is handled in ExtendTransitions()) so
// that the signed difference between a DateTime and the DateTime of
// its previous transition is always representable, without overflow.
// A contemporary zic will usually have already done this for us.
if (transitions_.empty() || transitions_.front().unix_time >= 0) {
Transition& tr(*transitions_.emplace(transitions_.begin()));
tr.unix_time = -(1LL << 59); // see tz/zic.c "BIG_BANG"
tr.type_index = default_transition_type_;
hdr.timecnt += 1;
}
// Extend the transitions using the future specification.
ExtendTransitions(name, hdr);
// Compute the local civil time for each transition and the preceeding
// second. These will be used for reverse conversions in MakeTime().
const TransitionType* ttp = &transition_types_[default_transition_type_];
for (std::size_t i = 0; i != transitions_.size(); ++i) {
Transition& tr(transitions_[i]);
tr.prev_date_time.Assign(LocalTime(tr.unix_time, *ttp).cs);
tr.prev_date_time.offset -= 1;
ttp = &transition_types_[tr.type_index];
tr.date_time.Assign(LocalTime(tr.unix_time, *ttp).cs);
if (i != 0) {
// Check that the transitions are ordered by date/time. Essentially
// this means that an offset change cannot cross another such change.
// No one does this in practice, and we depend on it in MakeTime().
if (!Transition::ByDateTime()(transitions_[i - 1], tr))
return false; // out of order
}
}
// We remember the transitions found during the last BreakTime() and
// MakeTime() calls. If the next request is for the same transition we
// will avoid re-searching.
local_time_hint_ = 0;
time_local_hint_ = 0;
transitions_.shrink_to_fit();
return true;
}
bool TimeZoneInfo::Load(const std::string& name) {
// We can ensure that the loading of UTC or any other fixed-offset
// zone never fails because the simple, fixed-offset state can be
// internally generated. Note that this depends on our choice to not
// accept leap-second encoded ("right") zoneinfo.
if (name == "UTC") return ResetToBuiltinUTC(0);
// Map time-zone name to its machine-specific path.
std::string path;
if (name == "localtime") {
#if defined(_MSC_VER)
char* localtime = nullptr;
_dupenv_s(&localtime, nullptr, "LOCALTIME");
path = localtime ? localtime : "/etc/localtime";
free(localtime);
#else
const char* localtime = std::getenv("LOCALTIME");
path = localtime ? localtime : "/etc/localtime";
#endif
} else if (!name.empty() && name[0] == '/') {
path = name;
} else {
#if defined(_MSC_VER)
char* tzdir = nullptr;
_dupenv_s(&tzdir, nullptr, "TZDIR");
path = tzdir ? tzdir : "/usr/share/zoneinfo";
free(tzdir);
#else
const char* tzdir = std::getenv("TZDIR");
path = tzdir ? tzdir : "/usr/share/zoneinfo";
#endif
path += '/';
path += name;
}
// Load the time-zone data.
bool loaded = false;
#if defined(_MSC_VER)
FILE* fp;
if (fopen_s(&fp, path.c_str(), "rb") != 0) fp = nullptr;
#else
FILE* fp = fopen(path.c_str(), "rb");
#endif
if (fp != nullptr) {
loaded = Load(name, fp);
fclose(fp);
} else {
char ebuf[64];
std::clog << path << ": " << errmsg(errno, ebuf, sizeof ebuf) << "\n";
}
return loaded;
}
// BreakTime() translation for a particular transition type.
time_zone::absolute_lookup TimeZoneInfo::LocalTime(
std::int_fast64_t unix_time, const TransitionType& tt) const {
time_zone::absolute_lookup al;
// A civil time in "+offset" looks like (time+offset) in UTC.
// Note: We perform two additions in the civil_second domain to
// sidestep the chance of overflow in (unix_time + tt.utc_offset).
al.cs = unix_epoch + unix_time;
al.cs += tt.utc_offset;
// Handle offset, is_dst, and abbreviation.
al.offset = tt.utc_offset;
al.is_dst = tt.is_dst;
al.abbr = &abbreviations_[tt.abbr_index];
return al;
}
// MakeTime() translation with a conversion-preserving offset.
time_zone::civil_lookup TimeZoneInfo::TimeLocal(
const civil_second& cs, std::int_fast64_t offset) const {
time_zone::civil_lookup cl = MakeTime(cs);
cl.pre += sys_seconds(offset);
cl.trans += sys_seconds(offset);
cl.post += sys_seconds(offset);
return cl;
}
time_zone::absolute_lookup TimeZoneInfo::BreakTime(
const time_point<sys_seconds>& tp) const {
std::int_fast64_t unix_time = ToUnixSeconds(tp);
const std::size_t timecnt = transitions_.size();
if (timecnt == 0 || unix_time < transitions_[0].unix_time) {
const std::uint_fast8_t type_index = default_transition_type_;
return LocalTime(unix_time, transition_types_[type_index]);
}
if (unix_time >= transitions_[timecnt - 1].unix_time) {
// After the last transition. If we extended the transitions using
// future_spec_, shift back to a supported year using the 400-year
// cycle of calendaric equivalence and then compensate accordingly.
if (extended_) {
const std::int_fast64_t diff =
unix_time - transitions_[timecnt - 1].unix_time;
const cctz::year_t shift = diff / kSecPer400Years + 1;
const auto d = sys_seconds(shift * kSecPer400Years);
time_zone::absolute_lookup al = BreakTime(tp - d);
al.cs = YearShift(al.cs, shift * 400);
return al;
}
const std::uint_fast8_t type_index = transitions_[timecnt - 1].type_index;
return LocalTime(unix_time, transition_types_[type_index]);
}
const std::size_t hint = local_time_hint_.load(std::memory_order_relaxed);
if (0 < hint && hint < timecnt) {
if (unix_time < transitions_[hint].unix_time) {
if (!(unix_time < transitions_[hint - 1].unix_time)) {
const std::uint_fast8_t type_index = transitions_[hint - 1].type_index;
return LocalTime(unix_time, transition_types_[type_index]);
}
}
}
const Transition target = {unix_time, 0, {0}, {0}};
const Transition* begin = &transitions_[0];
const Transition* tr = std::upper_bound(begin, begin + timecnt, target,
Transition::ByUnixTime());
local_time_hint_.store(tr - begin, std::memory_order_relaxed);
const std::uint_fast8_t type_index = (--tr)->type_index;
return LocalTime(unix_time, transition_types_[type_index]);
}
time_zone::civil_lookup TimeZoneInfo::MakeTime(const civil_second& cs) const {
Transition target;
DateTime& dt(target.date_time);
dt.Assign(cs);
const std::size_t timecnt = transitions_.size();
if (timecnt == 0) {
// Use the default offset.
const std::int_fast32_t default_offset =
transition_types_[default_transition_type_].utc_offset;
return MakeUnique((dt - DateTime{0}) - default_offset);
}
// Find the first transition after our target date/time.
const Transition* tr = nullptr;
const Transition* begin = &transitions_[0];
const Transition* end = begin + timecnt;
if (dt < begin->date_time) {
tr = begin;
} else if (!(dt < transitions_[timecnt - 1].date_time)) {
tr = end;
} else {
const std::size_t hint = time_local_hint_.load(std::memory_order_relaxed);
if (0 < hint && hint < timecnt) {
if (dt < transitions_[hint].date_time) {
if (!(dt < transitions_[hint - 1].date_time)) {
tr = begin + hint;
}
}
}
if (tr == nullptr) {
tr = std::upper_bound(begin, end, target, Transition::ByDateTime());
time_local_hint_.store(tr - begin, std::memory_order_relaxed);
}
}
if (tr == begin) {
if (!(tr->prev_date_time < dt)) {
// Before first transition, so use the default offset.
const std::int_fast32_t default_offset =
transition_types_[default_transition_type_].utc_offset;
return MakeUnique((dt - DateTime{0}) - default_offset);
}
// tr->prev_date_time < dt < tr->date_time
return MakeSkipped(*tr, dt);
}
if (tr == end) {
if ((--tr)->prev_date_time < dt) {
// After the last transition. If we extended the transitions using
// future_spec_, shift back to a supported year using the 400-year
// cycle of calendaric equivalence and then compensate accordingly.
if (extended_ && cs.year() > last_year_) {
const cctz::year_t shift = (cs.year() - last_year_) / 400 + 1;
return TimeLocal(YearShift(cs, shift * -400), shift * kSecPer400Years);
}
return MakeUnique(tr->unix_time + (dt - tr->date_time));
}
// tr->date_time <= dt <= tr->prev_date_time
return MakeRepeated(*tr, dt);
}
if (tr->prev_date_time < dt) {
// tr->prev_date_time < dt < tr->date_time
return MakeSkipped(*tr, dt);
}
if (!((--tr)->prev_date_time < dt)) {
// tr->date_time <= dt <= tr->prev_date_time
return MakeRepeated(*tr, dt);
}
// In between transitions.
return MakeUnique(tr->unix_time + (dt - tr->date_time));
}
} // namespace cctz

145
contrib/libcctz/src/time_zone_info.h Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CCTZ_TIME_ZONE_INFO_H_
#define CCTZ_TIME_ZONE_INFO_H_
#include <atomic>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
#include "civil_time.h"
#include "time_zone_if.h"
#include "tzfile.h"
namespace cctz {
// A zone-independent date/time. A DateTime represents a "Y/M/D H:M:S"
// as an offset in seconds from some epoch DateTime, without taking into
// account the value of, or changes in any time_zone's UTC offset (i.e., as
// if the date/time was in UTC). This allows "Y/M/D H:M:S" values to be
// quickly ordered by offset (although this may not be the same ordering as
// their corresponding times in a time_zone). But, if two DateTimes are not
// separated by a UTC-offset change in some time_zone, then the number of
// seconds between them can be computed as a simple difference of offsets.
//
// Note: Because the DateTime epoch does not correspond to the time_point
// epoch (even if only because of the unknown UTC offset) there can be valid
// times that will not be representable as DateTimes when DateTime only has
// the same number of "seconds" bits. We accept this at the moment (so as
// to avoid extended arithmetic) and lose a little range as a result.
struct DateTime {
std::int_least64_t offset; // seconds from some epoch DateTime
void Assign(const civil_second& cs);
};
inline bool operator<(const DateTime& lhs, const DateTime& rhs) {
return lhs.offset < rhs.offset;
}
// The difference between two DateTimes in seconds. Requires that all
// intervening DateTimes share the same UTC offset (i.e., no transitions).
inline std::int_fast64_t operator-(const DateTime& lhs, const DateTime& rhs) {
return lhs.offset - rhs.offset;
}
// A transition to a new UTC offset.
struct Transition {
std::int_least64_t unix_time; // the instant of this transition
std::uint_least8_t type_index; // index of the transition type
DateTime date_time; // local date/time of transition
DateTime prev_date_time; // local date/time one second earlier
struct ByUnixTime {
inline bool operator()(const Transition& lhs, const Transition& rhs) const {
return lhs.unix_time < rhs.unix_time;
}
};
struct ByDateTime {
inline bool operator()(const Transition& lhs, const Transition& rhs) const {
return lhs.date_time < rhs.date_time;
}
};
};
// The characteristics of a particular transition.
struct TransitionType {
std::int_least32_t utc_offset; // the new prevailing UTC offset
bool is_dst; // did we move into daylight-saving time
std::uint_least8_t abbr_index; // index of the new abbreviation
};
// A time zone backed by the IANA Time Zone Database (zoneinfo).
class TimeZoneInfo : public TimeZoneIf {
public:
TimeZoneInfo() = default;
TimeZoneInfo(const TimeZoneInfo&) = delete;
TimeZoneInfo& operator=(const TimeZoneInfo&) = delete;
// Loads the zoneinfo for the given name, returning true if successful.
bool Load(const std::string& name);
// TimeZoneIf implementations.
time_zone::absolute_lookup BreakTime(
const time_point<sys_seconds>& tp) const override;
time_zone::civil_lookup MakeTime(
const civil_second& cs) const override;
private:
struct Header { // counts of:
std::int_fast32_t timecnt; // transition times
std::int_fast32_t typecnt; // transition types
std::int_fast32_t charcnt; // zone abbreviation characters
std::int_fast32_t leapcnt; // leap seconds (we expect none)
std::int_fast32_t ttisstdcnt; // UTC/local indicators (unused)
std::int_fast32_t ttisgmtcnt; // standard/wall indicators (unused)
void Build(const tzhead& tzh);
std::size_t DataLength(std::size_t time_len) const;
};
void CheckTransition(const std::string& name, const TransitionType& tt,
std::int_fast32_t offset, bool is_dst,
const std::string& abbr) const;
bool EquivTransitions(std::uint_fast8_t tt1_index,
std::uint_fast8_t tt2_index) const;
void ExtendTransitions(const std::string& name, const Header& hdr);
bool ResetToBuiltinUTC(std::int_fast32_t seconds);
bool Load(const std::string& name, FILE* fp);
// Helpers for BreakTime() and MakeTime() respectively.
time_zone::absolute_lookup LocalTime(std::int_fast64_t unix_time,
const TransitionType& tt) const;
time_zone::civil_lookup TimeLocal(const civil_second& cs,
std::int_fast64_t offset) const;
std::vector<Transition> transitions_; // ordered by unix_time and date_time
std::vector<TransitionType> transition_types_; // distinct transition types
std::uint_fast8_t default_transition_type_; // for before first transition
std::string abbreviations_; // all the NUL-terminated abbreviations
std::string future_spec_; // for after the last zic transition
bool extended_; // future_spec_ was used to generate transitions
cctz::year_t last_year_; // the final year of the generated transitions
mutable std::atomic<std::size_t> local_time_hint_; // BreakTime() hint
mutable std::atomic<std::size_t> time_local_hint_; // MakeTime() hint
};
} // namespace cctz
#endif // CCTZ_TIME_ZONE_INFO_H_

118
contrib/libcctz/src/time_zone_libc.cc Normal file
View File

@ -0,0 +1,118 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "time_zone_libc.h"
#include <chrono>
#include <cstdint>
#include <ctime>
// Define OFFSET(tm) and ABBR(tm) for your platform to return the UTC
// offset and zone abbreviation after a call to localtime_r().
#if defined(linux)
# if defined(__USE_BSD) || defined(__USE_MISC)
# define OFFSET(tm) ((tm).tm_gmtoff)
# define ABBR(tm) ((tm).tm_zone)
# else
# define OFFSET(tm) ((tm).__tm_gmtoff)
# define ABBR(tm) ((tm).__tm_zone)
# endif
#elif defined(__APPLE__)
# define OFFSET(tm) ((tm).tm_gmtoff)
# define ABBR(tm) ((tm).tm_zone)
#elif defined(__sun)
# define OFFSET(tm) ((tm).tm_isdst > 0 ? altzone : timezone)
# define ABBR(tm) (tzname[(tm).tm_isdst > 0])
#elif defined(_WIN32) || defined(_WIN64)
static long get_timezone() {
long seconds;
_get_timezone(&seconds);
return seconds;
}
static std::string get_tzname(int index) {
char time_zone_name[32] = {0};
std::size_t size_in_bytes = sizeof time_zone_name;
_get_tzname(&size_in_bytes, time_zone_name, size_in_bytes, index);
return time_zone_name;
}
# define OFFSET(tm) (get_timezone() + ((tm).tm_isdst > 0 ? 60 * 60 : 0))
# define ABBR(tm) (get_tzname((tm).tm_isdst > 0))
#else
# define OFFSET(tm) (timezone + ((tm).tm_isdst > 0 ? 60 * 60 : 0))
# define ABBR(tm) (tzname[(tm).tm_isdst > 0])
#endif
namespace cctz {
TimeZoneLibC::TimeZoneLibC(const std::string& name) {
local_ = (name == "localtime");
if (!local_) {
// TODO: Support "UTC-05:00", for example.
offset_ = 0;
abbr_ = "UTC";
}
}
time_zone::absolute_lookup TimeZoneLibC::BreakTime(
const time_point<sys_seconds>& tp) const {
time_zone::absolute_lookup al;
std::time_t t = ToUnixSeconds(tp);
std::tm tm;
if (local_) {
#if defined(_WIN32) || defined(_WIN64)
localtime_s(&tm, &t);
#else
localtime_r(&t, &tm);
#endif
al.offset = OFFSET(tm);
al.abbr = ABBR(tm);
} else {
#if defined(_WIN32) || defined(_WIN64)
gmtime_s(&tm, &t);
#else
gmtime_r(&t, &tm);
#endif
al.offset = offset_;
al.abbr = abbr_;
}
// TODO: Eliminate redundant normalization.
al.cs = civil_second(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
tm.tm_hour, tm.tm_min, tm.tm_sec);
al.is_dst = tm.tm_isdst > 0;
return al;
}
time_zone::civil_lookup TimeZoneLibC::MakeTime(const civil_second& cs) const {
time_zone::civil_lookup cl;
std::time_t t;
if (local_) {
// Does not handle SKIPPED/AMBIGUOUS or huge years.
std::tm tm;
tm.tm_year = static_cast<int>(cs.year() - 1900);
tm.tm_mon = cs.month() - 1;
tm.tm_mday = cs.day();
tm.tm_hour = cs.hour();
tm.tm_min = cs.minute();
tm.tm_sec = cs.second();
tm.tm_isdst = -1;
t = std::mktime(&tm);
} else {
t = cs - civil_second();
}
cl.kind = time_zone::civil_lookup::UNIQUE;
cl.pre = cl.trans = cl.post = FromUnixSeconds(t);
return cl;
}
} // namespace cctz

45
contrib/libcctz/src/time_zone_libc.h Normal file
View File

@ -0,0 +1,45 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef CCTZ_TIME_ZONE_LIBC_H_
#define CCTZ_TIME_ZONE_LIBC_H_
#include <cstdint>
#include <string>
#include "time_zone_if.h"
namespace cctz {
// A time zone backed by gmtime_r(3), localtime_r(3), and mktime(3), and
// which therefore only supports "localtime" and fixed offsets from UTC.
class TimeZoneLibC : public TimeZoneIf {
public:
explicit TimeZoneLibC(const std::string& name);
// TimeZoneIf implementations.
time_zone::absolute_lookup BreakTime(
const time_point<sys_seconds>& tp) const override;
time_zone::civil_lookup MakeTime(
const civil_second& cs) const override;
private:
bool local_; // localtime or UTC
int offset_; // UTC offset when !local_
std::string abbr_; // abbreviation when !local_
};
} // namespace cctz
#endif // CCTZ_TIME_ZONE_LIBC_H_

69
contrib/libcctz/src/time_zone_lookup.cc Normal file
View File

@ -0,0 +1,69 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "time_zone.h"
#include <cstdlib>
#include "time_zone_impl.h"
namespace cctz {
std::string time_zone::name() const {
return time_zone::Impl::get(*this).name();
}
time_zone::absolute_lookup time_zone::lookup(
const time_point<sys_seconds>& tp) const {
return time_zone::Impl::get(*this).BreakTime(tp);
}
time_zone::civil_lookup time_zone::lookup(const civil_second& cs) const {
return time_zone::Impl::get(*this).MakeTime(cs);
}
bool operator==(time_zone lhs, time_zone rhs) {
return &time_zone::Impl::get(lhs) == &time_zone::Impl::get(rhs);
}
bool load_time_zone(const std::string& name, time_zone* tz) {
return time_zone::Impl::LoadTimeZone(name, tz);
}
time_zone utc_time_zone() {
return time_zone::Impl::UTC();
}
time_zone local_time_zone() {
#if defined(_MSC_VER)
char* tz_env = nullptr;
_dupenv_s(&tz_env, nullptr, "TZ");
const char* zone = tz_env;
#else
const char* zone = std::getenv("TZ");
#endif
if (zone != nullptr) {
if (*zone == ':') ++zone;
} else {
zone = "localtime";
}
time_zone tz;
load_time_zone(zone, &tz);
#if defined(_MSC_VER)
free(tz_env);
#endif
return tz;
}
} // namespace cctz

150
contrib/libcctz/src/time_zone_posix.cc Normal file
View File

@ -0,0 +1,150 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "time_zone_posix.h"
#include <cstring>
#include <limits>
#include <string>
namespace cctz {
namespace {
const char kDigits[] = "0123456789";
const char* ParseInt(const char* p, int min, int max, int* vp) {
int value = 0;
const char* op = p;
const int kMaxInt = std::numeric_limits<int>::max();
for (; const char* dp = strchr(kDigits, *p); ++p) {
int d = static_cast<int>(dp - kDigits);
if (d >= 10) break; // '\0'
if (value > kMaxInt / 10) return nullptr;
value *= 10;
if (value > kMaxInt - d) return nullptr;
value += d;
}
if (p == op || value < min || value > max) return nullptr;
*vp = value;
return p;
}
// abbr = <.*?> | [^-+,\d]{3,}
const char* ParseAbbr(const char* p, std::string* abbr) {
const char* op = p;
if (*p == '<') { // special zoneinfo <...> form
while (*++p != '>') {
if (*p == '\0') return nullptr;
}
abbr->assign(op + 1, p - op - 1);
return ++p;
}
while (*p != '\0') {
if (strchr("-+,", *p)) break;
if (strchr(kDigits, *p)) break;
++p;
}
if (p - op < 3) return nullptr;
abbr->assign(op, p - op);
return p;
}
// offset = [+|-]hh[:mm[:ss]] (aggregated into single seconds value)
const char* ParseOffset(const char* p, int min_hour, int max_hour, int sign,
std::int_fast32_t* offset) {
if (p == nullptr) return nullptr;
if (*p == '+' || *p == '-') {
if (*p++ == '-') sign = -sign;
}
int hours = 0;
int minutes = 0;
int seconds = 0;
p = ParseInt(p, min_hour, max_hour, &hours);
if (p == nullptr) return nullptr;
if (*p == ':') {
p = ParseInt(p + 1, 0, 59, &minutes);
if (p == nullptr) return nullptr;
if (*p == ':') {
p = ParseInt(p + 1, 0, 59, &seconds);
if (p == nullptr) return nullptr;
}
}
*offset = sign * ((((hours * 60) + minutes) * 60) + seconds);
return p;
}
// datetime = ( Jn | n | Mm.w.d ) [ / offset ]
const char* ParseDateTime(const char* p, PosixTransition* res) {
if (p != nullptr && *p == ',') {
if (*++p == 'M') {
int month = 0;
if ((p = ParseInt(p + 1, 1, 12, &month)) != nullptr && *p == '.') {
int week = 0;
if ((p = ParseInt(p + 1, 1, 5, &week)) != nullptr && *p == '.') {
int weekday = 0;
if ((p = ParseInt(p + 1, 0, 6, &weekday)) != nullptr) {
res->date.fmt = PosixTransition::M;
res->date.m.month = static_cast<int_fast8_t>(month);
res->date.m.week = static_cast<int_fast8_t>(week);
res->date.m.weekday = static_cast<int_fast8_t>(weekday);
}
}
}
} else if (*p == 'J') {
int day = 0;
if ((p = ParseInt(p + 1, 1, 365, &day)) != nullptr) {
res->date.fmt = PosixTransition::J;
res->date.j.day = static_cast<int_fast16_t>(day);
}
} else {
int day = 0;
if ((p = ParseInt(p, 0, 365, &day)) != nullptr) {
res->date.fmt = PosixTransition::N;
res->date.j.day = static_cast<int_fast16_t>(day);
}
}
}
if (p != nullptr) {
res->time.offset = 2 * 60 * 60; // default offset is 02:00:00
if (*p == '/') p = ParseOffset(p + 1, -167, 167, 1, &res->time.offset);
}
return p;
}
} // namespace
// spec = std offset [ dst [ offset ] , datetime , datetime ]
bool ParsePosixSpec(const std::string& spec, PosixTimeZone* res) {
const char* p = spec.c_str();
if (*p == ':') return false;
p = ParseAbbr(p, &res->std_abbr);
p = ParseOffset(p, 0, 24, -1, &res->std_offset);
if (p == nullptr) return false;
if (*p == '\0') return true;
p = ParseAbbr(p, &res->dst_abbr);
if (p == nullptr) return false;
res->dst_offset = res->std_offset + (60 * 60); // default
if (*p != ',') p = ParseOffset(p, 0, 24, -1, &res->dst_offset);
p = ParseDateTime(p, &res->dst_start);
p = ParseDateTime(p, &res->dst_end);
return p != nullptr && *p == '\0';
}
} // namespace cctz

114
contrib/libcctz/src/time_zone_posix.h Normal file
View File

@ -0,0 +1,114 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Parsing of a POSIX zone spec as described in the TZ part of section 8.3 in
// http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap08.html.
//
// The current POSIX spec for America/Los_Angeles is "PST8PDT,M3.2.0,M11.1.0",
// which would be broken down as ...
//
// PosixTimeZone {
// std_abbr = "PST"
// std_offset = -28800
// dst_abbr = "PDT"
// dst_offset = -25200
// dst_start = PosixTransition {
// date {
// m {
// month = 3
// week = 2
// weekday = 0
// }
// }
// time {
// offset = 7200
// }
// }
// dst_end = PosixTransition {
// date {
// m {
// month = 11
// week = 1
// weekday = 0
// }
// }
// time {
// offset = 7200
// }
// }
// }
#ifndef CCTZ_TIME_ZONE_POSIX_H_
#define CCTZ_TIME_ZONE_POSIX_H_
#include <cstdint>
#include <string>
namespace cctz {
// The date/time of the transition. The date is specified as either:
// (J) the Nth day of the year (1 <= N <= 365), excluding leap days, or
// (N) the Nth day of the year (0 <= N <= 365), including leap days, or
// (M) the Nth weekday of a month (e.g., the 2nd Sunday in March).
// The time, specified as a day offset, identifies the particular moment
// of the transition, and may be negative or >= 24h, and in which case
// it would take us to another day, and perhaps week, or even month.
struct PosixTransition {
enum DateFormat { J, N, M };
struct {
DateFormat fmt;
union {
struct {
std::int_fast16_t day; // day of non-leap year [1:365]
} j;
struct {
std::int_fast16_t day; // day of year [0:365]
} n;
struct {
std::int_fast8_t month; // month of year [1:12]
std::int_fast8_t week; // week of month [1:5] (5==last)
std::int_fast8_t weekday; // 0==Sun, ..., 6=Sat
} m;
};
} date;
struct {
std::int_fast32_t offset; // seconds before/after 00:00:00
} time;
};
// The entirety of a POSIX-string specified time-zone rule. The standard
// abbreviation and offset are always given. If the time zone includes
// daylight saving, then the daylight abbrevation is non-empty and the
// remaining fields are also valid. Note that the start/end transitions
// are not ordered---in the southern hemisphere the transition to end
// daylight time occurs first in any particular year.
struct PosixTimeZone {
std::string std_abbr;
std::int_fast32_t std_offset;
std::string dst_abbr;
std::int_fast32_t dst_offset;
PosixTransition dst_start;
PosixTransition dst_end;
};
// Breaks down a POSIX time-zone specification into its constituent pieces,
// filling in any missing values (DST offset, or start/end transition times)
// with the standard-defined defaults. Returns false if the specification
// could not be parsed (although some fields of *res may have been altered).
bool ParsePosixSpec(const std::string& spec, PosixTimeZone* res);
} // namespace cctz
#endif // CCTZ_TIME_ZONE_POSIX_H_

169
contrib/libcctz/src/tzfile.h Normal file
View File

@ -0,0 +1,169 @@
#ifndef TZFILE_H
#define TZFILE_H
/*
** This file is in the public domain, so clarified as of
** 1996-06-05 by Arthur David Olson.
*/
/*
** This header is for use ONLY with the time conversion code.
** There is no guarantee that it will remain unchanged,
** or that it will remain at all.
** Do NOT copy it to any system include directory.
** Thank you!
*/
/*
** Information about time zone files.
*/
#ifndef TZDIR
#define TZDIR "/usr/local/etc/zoneinfo" /* Time zone object file directory */
#endif /* !defined TZDIR */
#ifndef TZDEFAULT
#define TZDEFAULT "localtime"
#endif /* !defined TZDEFAULT */
#ifndef TZDEFRULES
#define TZDEFRULES "posixrules"
#endif /* !defined TZDEFRULES */
/*
** Each file begins with. . .
*/
#define TZ_MAGIC "TZif"
struct tzhead {
char tzh_magic[4]; /* TZ_MAGIC */
char tzh_version[1]; /* '\0' or '2' or '3' as of 2013 */
char tzh_reserved[15]; /* reserved; must be zero */
char tzh_ttisgmtcnt[4]; /* coded number of trans. time flags */
char tzh_ttisstdcnt[4]; /* coded number of trans. time flags */
char tzh_leapcnt[4]; /* coded number of leap seconds */
char tzh_timecnt[4]; /* coded number of transition times */
char tzh_typecnt[4]; /* coded number of local time types */
char tzh_charcnt[4]; /* coded number of abbr. chars */
};
/*
** . . .followed by. . .
**
** tzh_timecnt (char [4])s coded transition times a la time(2)
** tzh_timecnt (unsigned char)s types of local time starting at above
** tzh_typecnt repetitions of
** one (char [4]) coded UT offset in seconds
** one (unsigned char) used to set tm_isdst
** one (unsigned char) that's an abbreviation list index
** tzh_charcnt (char)s '\0'-terminated zone abbreviations
** tzh_leapcnt repetitions of
** one (char [4]) coded leap second transition times
** one (char [4]) total correction after above
** tzh_ttisstdcnt (char)s indexed by type; if 1, transition
** time is standard time, if 0,
** transition time is wall clock time
** if absent, transition times are
** assumed to be wall clock time
** tzh_ttisgmtcnt (char)s indexed by type; if 1, transition
** time is UT, if 0,
** transition time is local time
** if absent, transition times are
** assumed to be local time
*/
/*
** If tzh_version is '2' or greater, the above is followed by a second instance
** of tzhead and a second instance of the data in which each coded transition
** time uses 8 rather than 4 chars,
** then a POSIX-TZ-environment-variable-style string for use in handling
** instants after the last transition time stored in the file
** (with nothing between the newlines if there is no POSIX representation for
** such instants).
**
** If tz_version is '3' or greater, the above is extended as follows.
** First, the POSIX TZ string's hour offset may range from -167
** through 167 as compared to the POSIX-required 0 through 24.
** Second, its DST start time may be January 1 at 00:00 and its stop
** time December 31 at 24:00 plus the difference between DST and
** standard time, indicating DST all year.
*/
/*
** In the current implementation, "tzset()" refuses to deal with files that
** exceed any of the limits below.
*/
#ifndef TZ_MAX_TIMES
#define TZ_MAX_TIMES 2000
#endif /* !defined TZ_MAX_TIMES */
#ifndef TZ_MAX_TYPES
/* This must be at least 17 for Europe/Samara and Europe/Vilnius. */
#define TZ_MAX_TYPES 256 /* Limited by what (unsigned char)'s can hold */
#endif /* !defined TZ_MAX_TYPES */
#ifndef TZ_MAX_CHARS
#define TZ_MAX_CHARS 50 /* Maximum number of abbreviation characters */
/* (limited by what unsigned chars can hold) */
#endif /* !defined TZ_MAX_CHARS */
#ifndef TZ_MAX_LEAPS
#define TZ_MAX_LEAPS 50 /* Maximum number of leap second corrections */
#endif /* !defined TZ_MAX_LEAPS */
#define SECSPERMIN 60
#define MINSPERHOUR 60
#define HOURSPERDAY 24
#define DAYSPERWEEK 7
#define DAYSPERNYEAR 365
#define DAYSPERLYEAR 366
#define SECSPERHOUR (SECSPERMIN * MINSPERHOUR)
#define SECSPERDAY ((int_fast32_t) SECSPERHOUR * HOURSPERDAY)
#define MONSPERYEAR 12
#define TM_SUNDAY 0
#define TM_MONDAY 1
#define TM_TUESDAY 2
#define TM_WEDNESDAY 3
#define TM_THURSDAY 4
#define TM_FRIDAY 5
#define TM_SATURDAY 6
#define TM_JANUARY 0
#define TM_FEBRUARY 1
#define TM_MARCH 2
#define TM_APRIL 3
#define TM_MAY 4
#define TM_JUNE 5
#define TM_JULY 6
#define TM_AUGUST 7
#define TM_SEPTEMBER 8
#define TM_OCTOBER 9
#define TM_NOVEMBER 10
#define TM_DECEMBER 11
#define TM_YEAR_BASE 1900
#define EPOCH_YEAR 1970
#define EPOCH_WDAY TM_THURSDAY
#define isleap(y) (((y) % 4) == 0 && (((y) % 100) != 0 || ((y) % 400) == 0))
/*
** Since everything in isleap is modulo 400 (or a factor of 400), we know that
** isleap(y) == isleap(y % 400)
** and so
** isleap(a + b) == isleap((a + b) % 400)
** or
** isleap(a + b) == isleap(a % 400 + b % 400)
** This is true even if % means modulo rather than Fortran remainder
** (which is allowed by C89 but not C99).
** We use this to avoid addition overflow problems.
*/
#define isleap_sum(a, b) isleap((a) % 400 + (b) % 400)
#endif /* !defined TZFILE_H */

2
contrib/liblz4/CMakeLists.txt
View File

@ -1,3 +1,5 @@
include_directories (BEFORE include)
add_library (lz4
src/lz4.c
src/lz4hc.c

3
contrib/libpoco/CMakeLists.txt
View File

@ -134,6 +134,9 @@ endif(WIN32)
if (UNIX AND NOT ANDROID )
add_definitions( -DPOCO_OS_FAMILY_UNIX )
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field -Wno-unused-local-typedef -Wno-for-loop-analysis -Wno-unknown-pragmas -Wno-unused-variable")
if (APPLE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
endif ()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unknown-pragmas -Wno-unused-variable")
# Standard 'must be' defines
if (APPLE)

4
contrib/libpoco/Net/cmake/test_anl.cmake
View File

@ -14,10 +14,10 @@ endif ()
find_library (ANL_LIB NAMES ${ANL_LIB_NAME})
# better use Threads::Threads but incompatible with cmake < 3
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
if (ANL_LIB)
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${ANL_LIB})
endif ()
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
check_cxx_source_runs("
#include <netdb.h>
@ -27,7 +27,7 @@ check_cxx_source_runs("
}
" HAVE_GETADDRINFO_A)
#message(STATUS "test_anl: USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} ANL_LIB_NAME=${ANL_LIB_NAME} ANL_LIB=${ANL_LIB} HAVE_GETADDRINFO_A=${HAVE_GETADDRINFO_A}")
#message(STATUS "test_anl: USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} ANL_LIB_NAME=${ANL_LIB_NAME} ANL_LIB=${ANL_LIB} HAVE_GETADDRINFO_A=${HAVE_GETADDRINFO_A} CMAKE_REQUIRED_LIBRARIES=${CMAKE_REQUIRED_LIBRARIES}")
if (HAVE_GETADDRINFO_A)
add_definitions (-DHAVE_GETADDRINFO_A=1)

28
contrib/libpoco/Net/include/Poco/Net/HTTPResponse.h
View File

@ -90,7 +90,7 @@ public:
HTTPResponse();
/// Creates the HTTPResponse with OK status.
HTTPResponse(HTTPStatus status, const std::string& reason);
/// Creates the HTTPResponse with the given status
/// and reason phrase.
@ -98,7 +98,7 @@ public:
HTTPResponse(const std::string& version, HTTPStatus status, const std::string& reason);
/// Creates the HTTPResponse with the given version, status
/// and reason phrase.
HTTPResponse(HTTPStatus status);
/// Creates the HTTPResponse with the given status
/// an an appropriate reason phrase.
@ -114,25 +114,25 @@ public:
/// Sets the HTTP status code.
///
/// Does not change the reason phrase.
HTTPStatus getStatus() const;
/// Returns the HTTP status code.
void setStatus(const std::string& status);
/// Sets the HTTP status code.
///
/// The string must contain a valid
/// HTTP numerical status code.
void setReason(const std::string& reason);
/// Sets the HTTP reason phrase.
const std::string& getReason() const;
/// Returns the HTTP reason phrase.
void setStatusAndReason(HTTPStatus status, const std::string& reason);
/// Sets the HTTP status code and reason phrase.
void setStatusAndReason(HTTPStatus status);
/// Sets the HTTP status code and reason phrase.
///
@ -140,7 +140,7 @@ public:
void setDate(const Poco::Timestamp& dateTime);
/// Sets the Date header to the given date/time value.
Poco::Timestamp getDate() const;
/// Returns the value of the Date header.
@ -159,12 +159,16 @@ public:
/// Writes the HTTP response to the given
/// output stream.
void beginWrite(std::ostream& ostr) const;
/// Writes the HTTP response to the given
/// output stream, but do not finish with \r\n delimiter.
void read(std::istream& istr);
/// Reads the HTTP response from the
/// given input stream.
///
/// 100 Continue responses are ignored.
static const std::string& getReasonForStatus(HTTPStatus status);
/// Returns an appropriate reason phrase
/// for the given status code.
@ -210,7 +214,7 @@ public:
static const std::string HTTP_REASON_GATEWAY_TIMEOUT;
static const std::string HTTP_REASON_VERSION_NOT_SUPPORTED;
static const std::string HTTP_REASON_UNKNOWN;
static const std::string DATE;
static const std::string SET_COOKIE;
@ -221,10 +225,10 @@ private:
MAX_STATUS_LENGTH = 3,
MAX_REASON_LENGTH = 512
};
HTTPStatus _status;
std::string _reason;
HTTPResponse(const HTTPResponse&);
HTTPResponse& operator = (const HTTPResponse&);
};

31
contrib/libpoco/Net/include/Poco/Net/HTTPServerResponse.h
View File

@ -55,7 +55,7 @@ public:
virtual void sendContinue() = 0;
/// Sends a 100 Continue response to the
/// client.
virtual std::ostream& send() = 0;
/// Sends the response header to the client and
/// returns an output stream for sending the
@ -64,20 +64,31 @@ public:
/// The returned stream is valid until the response
/// object is destroyed.
///
/// Must not be called after sendFile(), sendBuffer()
/// Must not be called after beginSend(), sendFile(), sendBuffer()
/// or redirect() has been called.
virtual std::pair<std::ostream *, std::ostream *> beginSend() = 0;
/// Sends the response headers to the client
/// but do not finish headers with \r\n,
/// allowing to continue sending additional header fields.
///
/// Returns an output streams for sending the remaining headers
/// and response body.
///
/// Must not be called after send(), sendFile(), sendBuffer()
/// or redirect() has been called.
virtual void sendFile(const std::string& path, const std::string& mediaType) = 0;
/// Sends the response header to the client, followed
/// by the content of the given file.
///
/// Must not be called after send(), sendBuffer()
/// Must not be called after send(), sendBuffer()
/// or redirect() has been called.
///
/// Throws a FileNotFoundException if the file
/// cannot be found, or an OpenFileException if
/// the file cannot be opened.
virtual void sendBuffer(const void* pBuffer, std::size_t length) = 0;
/// Sends the response header to the client, followed
/// by the contents of the given buffer.
@ -86,12 +97,12 @@ public:
/// to length and chunked transfer encoding is disabled.
///
/// If both the HTTP message header and body (from the
/// given buffer) fit into one single network packet, the
/// given buffer) fit into one single network packet, the
/// complete response can be sent in one network packet.
///
/// Must not be called after send(), sendFile()
/// Must not be called after send(), sendFile()
/// or redirect() has been called.
virtual void redirect(const std::string& uri, HTTPStatus status = HTTP_FOUND) = 0;
/// Sets the status code, which must be one of
/// HTTP_MOVED_PERMANENTLY (301), HTTP_FOUND (302),
@ -101,12 +112,12 @@ public:
/// the HTTP specification, must be absolute.
///
/// Must not be called after send() has been called.
virtual void requireAuthentication(const std::string& realm) = 0;
/// Sets the status code to 401 (Unauthorized)
/// and sets the "WWW-Authenticate" header field
/// according to the given realm.
virtual bool sent() const = 0;
/// Returns true if the response (header) has been sent.
};

36
contrib/libpoco/Net/include/Poco/Net/HTTPServerResponseImpl.h
View File

@ -53,7 +53,7 @@ public:
void sendContinue();
/// Sends a 100 Continue response to the
/// client.
std::ostream& send();
/// Sends the response header to the client and
/// returns an output stream for sending the
@ -62,20 +62,31 @@ public:
/// The returned stream is valid until the response
/// object is destroyed.
///
/// Must not be called after sendFile(), sendBuffer()
/// Must not be called after beginSend(), sendFile(), sendBuffer()
/// or redirect() has been called.
std::pair<std::ostream *, std::ostream *> beginSend();
/// Sends the response headers to the client
/// but do not finish headers with \r\n,
/// allowing to continue sending additional header fields.
///
/// Returns an output streams for sending the remaining headers
/// and response body.
///
/// Must not be called after send(), sendFile(), sendBuffer()
/// or redirect() has been called.
void sendFile(const std::string& path, const std::string& mediaType);
/// Sends the response header to the client, followed
/// by the content of the given file.
///
/// Must not be called after send(), sendBuffer()
/// Must not be called after send(), sendBuffer()
/// or redirect() has been called.
///
/// Throws a FileNotFoundException if the file
/// cannot be found, or an OpenFileException if
/// the file cannot be opened.
void sendBuffer(const void* pBuffer, std::size_t length);
/// Sends the response header to the client, followed
/// by the contents of the given buffer.
@ -84,12 +95,12 @@ public:
/// to length and chunked transfer encoding is disabled.
///
/// If both the HTTP message header and body (from the
/// given buffer) fit into one single network packet, the
/// given buffer) fit into one single network packet, the
/// complete response can be sent in one network packet.
///
/// Must not be called after send(), sendFile()
/// Must not be called after send(), sendFile()
/// or redirect() has been called.
void redirect(const std::string& uri, HTTPStatus status = HTTP_FOUND);
/// Sets the status code, which must be one of
/// HTTP_MOVED_PERMANENTLY (301), HTTP_FOUND (302),
@ -99,23 +110,24 @@ public:
/// the HTTP specification, must be absolute.
///
/// Must not be called after send() has been called.
void requireAuthentication(const std::string& realm);
/// Sets the status code to 401 (Unauthorized)
/// and sets the "WWW-Authenticate" header field
/// according to the given realm.
bool sent() const;
/// Returns true if the response (header) has been sent.
protected:
void attachRequest(HTTPServerRequestImpl* pRequest);
private:
HTTPServerSession& _session;
HTTPServerRequestImpl* _pRequest;
std::ostream* _pStream;
std::ostream* _pHeaderStream;
friend class HTTPServerRequestImpl;
};

102
contrib/libpoco/Net/src/HTTPResponse.cpp
View File

@ -89,7 +89,7 @@ HTTPResponse::HTTPResponse():
{
}
HTTPResponse::HTTPResponse(HTTPStatus status, const std::string& reason):
_status(status),
_reason(reason)
@ -97,7 +97,7 @@ HTTPResponse::HTTPResponse(HTTPStatus status, const std::string& reason):
}
HTTPResponse::HTTPResponse(const std::string& version, HTTPStatus status, const std::string& reason):
HTTPMessage(version),
_status(status),
@ -105,7 +105,7 @@ HTTPResponse::HTTPResponse(const std::string& version, HTTPStatus status, const
{
}
HTTPResponse::HTTPResponse(HTTPStatus status):
_status(status),
_reason(getReasonForStatus(status))
@ -136,8 +136,8 @@ void HTTPResponse::setStatus(const std::string& status)
{
setStatus((HTTPStatus) NumberParser::parse(status));
}
void HTTPResponse::setReason(const std::string& reason)
{
_reason = reason;
@ -150,7 +150,7 @@ void HTTPResponse::setStatusAndReason(HTTPStatus status, const std::string& reas
_reason = reason;
}
void HTTPResponse::setStatusAndReason(HTTPStatus status)
{
setStatusAndReason(status, getReasonForStatus(status));
@ -162,7 +162,7 @@ void HTTPResponse::setDate(const Poco::Timestamp& dateTime)
set(DATE, DateTimeFormatter::format(dateTime, DateTimeFormat::HTTP_FORMAT));
}
Poco::Timestamp HTTPResponse::getDate() const
{
const std::string& dateTime = get(DATE);
@ -192,10 +192,16 @@ void HTTPResponse::getCookies(std::vector<HTTPCookie>& cookies) const
void HTTPResponse::write(std::ostream& ostr) const
{
beginWrite(ostr);
ostr << "\r\n";
}
void HTTPResponse::beginWrite(std::ostream& ostr) const
{
ostr << getVersion() << " " << static_cast<int>(_status) << " " << _reason << "\r\n";
HTTPMessage::write(ostr);
ostr << "\r\n";
}
@ -206,7 +212,7 @@ void HTTPResponse::read(std::istream& istr)
std::string version;
std::string status;
std::string reason;
int ch = istr.get();
if (istr.bad()) throw NetException("Error reading HTTP response header");
if (ch == eof) throw NoMessageException();
@ -235,87 +241,87 @@ const std::string& HTTPResponse::getReasonForStatus(HTTPStatus status)
{
switch (status)
{
case HTTP_CONTINUE:
case HTTP_CONTINUE:
return HTTP_REASON_CONTINUE;
case HTTP_SWITCHING_PROTOCOLS:
case HTTP_SWITCHING_PROTOCOLS:
return HTTP_REASON_SWITCHING_PROTOCOLS;
case HTTP_OK:
case HTTP_OK:
return HTTP_REASON_OK;
case HTTP_CREATED:
case HTTP_CREATED:
return HTTP_REASON_CREATED;
case HTTP_ACCEPTED:
case HTTP_ACCEPTED:
return HTTP_REASON_ACCEPTED;
case HTTP_NONAUTHORITATIVE:
case HTTP_NONAUTHORITATIVE:
return HTTP_REASON_NONAUTHORITATIVE;
case HTTP_NO_CONTENT:
case HTTP_NO_CONTENT:
return HTTP_REASON_NO_CONTENT;
case HTTP_RESET_CONTENT:
case HTTP_RESET_CONTENT:
return HTTP_REASON_RESET_CONTENT;
case HTTP_PARTIAL_CONTENT:
case HTTP_PARTIAL_CONTENT:
return HTTP_REASON_PARTIAL_CONTENT;
case HTTP_MULTIPLE_CHOICES:
case HTTP_MULTIPLE_CHOICES:
return HTTP_REASON_MULTIPLE_CHOICES;
case HTTP_MOVED_PERMANENTLY:
case HTTP_MOVED_PERMANENTLY:
return HTTP_REASON_MOVED_PERMANENTLY;
case HTTP_FOUND:
case HTTP_FOUND:
return HTTP_REASON_FOUND;
case HTTP_SEE_OTHER:
case HTTP_SEE_OTHER:
return HTTP_REASON_SEE_OTHER;
case HTTP_NOT_MODIFIED:
case HTTP_NOT_MODIFIED:
return HTTP_REASON_NOT_MODIFIED;
case HTTP_USEPROXY:
case HTTP_USEPROXY:
return HTTP_REASON_USEPROXY;
case HTTP_TEMPORARY_REDIRECT:
case HTTP_TEMPORARY_REDIRECT:
return HTTP_REASON_TEMPORARY_REDIRECT;
case HTTP_BAD_REQUEST:
case HTTP_BAD_REQUEST:
return HTTP_REASON_BAD_REQUEST;
case HTTP_UNAUTHORIZED:
case HTTP_UNAUTHORIZED:
return HTTP_REASON_UNAUTHORIZED;
case HTTP_PAYMENT_REQUIRED:
case HTTP_PAYMENT_REQUIRED:
return HTTP_REASON_PAYMENT_REQUIRED;
case HTTP_FORBIDDEN:
case HTTP_FORBIDDEN:
return HTTP_REASON_FORBIDDEN;
case HTTP_NOT_FOUND:
case HTTP_NOT_FOUND:
return HTTP_REASON_NOT_FOUND;
case HTTP_METHOD_NOT_ALLOWED:
return HTTP_REASON_METHOD_NOT_ALLOWED;
case HTTP_NOT_ACCEPTABLE:
case HTTP_NOT_ACCEPTABLE:
return HTTP_REASON_NOT_ACCEPTABLE;
case HTTP_PROXY_AUTHENTICATION_REQUIRED:
case HTTP_PROXY_AUTHENTICATION_REQUIRED:
return HTTP_REASON_PROXY_AUTHENTICATION_REQUIRED;
case HTTP_REQUEST_TIMEOUT:
case HTTP_REQUEST_TIMEOUT:
return HTTP_REASON_REQUEST_TIMEOUT;
case HTTP_CONFLICT:
case HTTP_CONFLICT:
return HTTP_REASON_CONFLICT;
case HTTP_GONE:
case HTTP_GONE:
return HTTP_REASON_GONE;
case HTTP_LENGTH_REQUIRED:
case HTTP_LENGTH_REQUIRED:
return HTTP_REASON_LENGTH_REQUIRED;
case HTTP_PRECONDITION_FAILED:
case HTTP_PRECONDITION_FAILED:
return HTTP_REASON_PRECONDITION_FAILED;
case HTTP_REQUESTENTITYTOOLARGE:
case HTTP_REQUESTENTITYTOOLARGE:
return HTTP_REASON_REQUESTENTITYTOOLARGE;
case HTTP_REQUESTURITOOLONG:
case HTTP_REQUESTURITOOLONG:
return HTTP_REASON_REQUESTURITOOLONG;
case HTTP_UNSUPPORTEDMEDIATYPE:
case HTTP_UNSUPPORTEDMEDIATYPE:
return HTTP_REASON_UNSUPPORTEDMEDIATYPE;
case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
return HTTP_REASON_REQUESTED_RANGE_NOT_SATISFIABLE;
case HTTP_EXPECTATION_FAILED:
case HTTP_EXPECTATION_FAILED:
return HTTP_REASON_EXPECTATION_FAILED;
case HTTP_INTERNAL_SERVER_ERROR:
case HTTP_INTERNAL_SERVER_ERROR:
return HTTP_REASON_INTERNAL_SERVER_ERROR;
case HTTP_NOT_IMPLEMENTED:
case HTTP_NOT_IMPLEMENTED:
return HTTP_REASON_NOT_IMPLEMENTED;
case HTTP_BAD_GATEWAY:
case HTTP_BAD_GATEWAY:
return HTTP_REASON_BAD_GATEWAY;
case HTTP_SERVICE_UNAVAILABLE:
return HTTP_REASON_SERVICE_UNAVAILABLE;
case HTTP_GATEWAY_TIMEOUT:
case HTTP_GATEWAY_TIMEOUT:
return HTTP_REASON_GATEWAY_TIMEOUT;
case HTTP_VERSION_NOT_SUPPORTED:
case HTTP_VERSION_NOT_SUPPORTED:
return HTTP_REASON_VERSION_NOT_SUPPORTED;
default:
default:
return HTTP_REASON_UNKNOWN;
}
}

52
contrib/libpoco/Net/src/HTTPServerResponseImpl.cpp
View File

@ -48,14 +48,18 @@ namespace Net {
HTTPServerResponseImpl::HTTPServerResponseImpl(HTTPServerSession& session):
_session(session),
_pRequest(0),
_pStream(0)
_pStream(0),
_pHeaderStream(0)
{
}
HTTPServerResponseImpl::~HTTPServerResponseImpl()
{
delete _pStream;
if (_pHeaderStream && _pHeaderStream != _pStream)
delete _pHeaderStream;
if (_pStream)
delete _pStream;
}
@ -90,7 +94,7 @@ std::ostream& HTTPServerResponseImpl::send()
{
Poco::CountingOutputStream cs;
write(cs);
#if defined(POCO_HAVE_INT64)
#if defined(POCO_HAVE_INT64)
_pStream = new HTTPFixedLengthOutputStream(_session, getContentLength64() + cs.chars());
#else
_pStream = new HTTPFixedLengthOutputStream(_session, getContentLength() + cs.chars());
@ -107,6 +111,42 @@ std::ostream& HTTPServerResponseImpl::send()
}
std::pair<std::ostream *, std::ostream *> HTTPServerResponseImpl::beginSend()
{
poco_assert (!_pStream);
poco_assert (!_pHeaderStream);
// NOTE Code is not exception safe.
if ((_pRequest && _pRequest->getMethod() == HTTPRequest::HTTP_HEAD) ||
getStatus() < 200 ||
getStatus() == HTTPResponse::HTTP_NO_CONTENT ||
getStatus() == HTTPResponse::HTTP_NOT_MODIFIED)
{
throw Exception("HTTPServerResponse::beginSend is invalid for HEAD request");
}
else if (getChunkedTransferEncoding())
{
_pHeaderStream = new HTTPHeaderOutputStream(_session);
beginWrite(*_pHeaderStream);
_pStream = new HTTPChunkedOutputStream(_session);
}
else if (hasContentLength())
{
throw Exception("HTTPServerResponse::beginSend is invalid for response with Content-Length header");
}
else
{
_pStream = new HTTPOutputStream(_session);
_pHeaderStream = _pStream;
setKeepAlive(false);
beginWrite(*_pStream);
}
return std::make_pair(_pHeaderStream, _pStream);
}
void HTTPServerResponseImpl::sendFile(const std::string& path, const std::string& mediaType)
{
poco_assert (!_pStream);
@ -115,7 +155,7 @@ void HTTPServerResponseImpl::sendFile(const std::string& path, const std::string
Timestamp dateTime = f.getLastModified();
File::FileSize length = f.getSize();
set("Last-Modified", DateTimeFormatter::format(dateTime, DateTimeFormat::HTTP_FORMAT));
#if defined(POCO_HAVE_INT64)
#if defined(POCO_HAVE_INT64)
setContentLength64(length);
#else
setContentLength(static_cast<int>(length));
@ -143,7 +183,7 @@ void HTTPServerResponseImpl::sendBuffer(const void* pBuffer, std::size_t length)
setContentLength(static_cast<int>(length));
setChunkedTransferEncoding(false);
_pStream = new HTTPHeaderOutputStream(_session);
write(*_pStream);
if (_pRequest && _pRequest->getMethod() != HTTPRequest::HTTP_HEAD)
@ -171,7 +211,7 @@ void HTTPServerResponseImpl::redirect(const std::string& uri, HTTPStatus status)
void HTTPServerResponseImpl::requireAuthentication(const std::string& realm)
{
poco_assert (!_pStream);
setStatusAndReason(HTTPResponse::HTTP_UNAUTHORIZED);
std::string auth("Basic realm=\"");
auth.append(realm);

2
contrib/libre2/CMakeLists.txt
View File

@ -38,6 +38,8 @@ set (re2_sources
# In order to avoid redundant locks in some cases, we use not thread-safe version of the library (re2_st).
add_definitions (-DNDEBUG)
include_directories (BEFORE .)
add_library (re2 ${re2_sources})
add_library (re2_st ${re2_sources})

2
contrib/libtcmalloc/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_definitions(
-DNO_TCMALLOC_SAMPLES -DNO_TCMALLOC_SAMPLES
-DNO_TCMALLOC_SAMPLES
-DNDEBUG -DNO_FRAME_POINTER
-Wwrite-strings -Wno-sign-compare -Wno-unused-result
-Wno-deprecated-declarations -Wno-unused-function

4
contrib/libzlib-ng/CMakeLists.txt
View File

@ -540,6 +540,10 @@ endif()
# Example binaries
#============================================================================
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_EXE_LINKER_FLAGS "")
endif ()
add_executable(example test/example.c)
target_link_libraries(example zlib)
add_test(example example)

2
contrib/libzlib-ng/inflate.c
View File

@ -1460,7 +1460,7 @@ long ZEXPORT inflateMark(z_stream *strm) {
struct inflate_state *state;
if (strm == Z_NULL || strm->state == Z_NULL)
return -1L << 16;
return 0xFFFFFFFFFFFF0000L;
state = (struct inflate_state *)strm->state;
return ((long)(state->back) << 16) + (state->mode == COPY ? state->length :
(state->mode == MATCH ? state->was - state->length : 0));

5
contrib/libzookeeper/src/mt_adaptor.c
View File

@ -484,7 +484,10 @@ int32_t inc_ref_counter(zhandle_t* zh,int i)
int32_t fetch_and_add(volatile int32_t* operand, int incr)
{
#ifndef WIN32
#if defined(__ARM_ARCH)
// this is gcc-specific func https://gcc.gnu.org/onlinedocs/gcc-6.3.0/gcc/_005f_005fatomic-Builtins.html
return __atomic_fetch_add(operand, incr, __ATOMIC_RELAXED);
#elif !defined(WIN32)
int32_t result;
asm __volatile__(
"lock xaddl %0,%1\n"

21
dbms/CMakeLists.txt
View File

@ -1,18 +1,27 @@
include_directories (BEFORE include)
#include_directories (/usr/include/mysql)
include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
add_subdirectory (src)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libsparsehash)
include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libre2/)
include_directories (BEFORE ${ClickHouse_BINARY_DIR}/contrib/libre2/)
include_directories (${ClickHouse_SOURCE_DIR}/libs/libdaemon/include/)
if (NOT ENABLE_LIBTCMALLOC)
add_definitions(-D NO_TCMALLOC)
if (NOT NO_WERROR)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
endif ()
add_subdirectory (src)
add_library(string_utils
include/DB/Common/StringUtils.h
src/Common/StringUtils.cpp)
set(dbms_headers)
set(dbms_sources)

7
dbms/include/DB/AggregateFunctions/AggregateFunctionFactory.h
View File

@ -1,12 +1,17 @@
#pragma once
#include <unordered_map>
#include <DB/AggregateFunctions/IAggregateFunction.h>
#include <DB/DataTypes/IDataType.h>
namespace DB
{
class IDataType;
using DataTypePtr = std::shared_ptr<IDataType>;
using DataTypes = std::vector<DataTypePtr>;
/** Creates an aggregate function by name.
*/
class AggregateFunctionFactory final

4
dbms/include/DB/Analyzers/AnalyzeArrayJoins.h
View File

@ -9,8 +9,8 @@ namespace DB
{
class WriteBuffer;
class CollectAliases;
class CollectTables;
struct CollectAliases;
struct CollectTables;
/** For every ARRAY JOIN, collect a map:

4
dbms/include/DB/Analyzers/AnalyzeColumns.h
View File

@ -9,8 +9,8 @@ namespace DB
{
class WriteBuffer;
class CollectAliases;
class CollectTables;
struct CollectAliases;
struct CollectTables;
/** For every identifier, that is not an alias,

35
dbms/include/DB/Analyzers/AnalyzeLambdas.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#include <DB/Parsers/IAST.h>
#include <vector>
namespace DB
{
class WriteBuffer;
/** For every lambda expression, rename its parameters to '_lambda0_arg0' form.
* Check correctness of lambda expressions.
* Find functions, that have lambda expressions as arguments (they are called "higher order" functions).
*
* This should be done before CollectAliases.
*/
struct AnalyzeLambdas
{
void process(ASTPtr & ast);
/// Parameters of lambda expressions.
using LambdaParameters = std::vector<String>;
static LambdaParameters extractLambdaParameters(ASTPtr & ast);
using HigherOrderFunctions = std::vector<ASTPtr>;
HigherOrderFunctions higher_order_functions;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

2
dbms/include/DB/Analyzers/CollectTables.h
View File

@ -9,7 +9,7 @@ namespace DB
{
class Context;
class CollectAliases;
struct CollectAliases;
class WriteBuffer;

2
dbms/include/DB/Analyzers/OptimizeGroupOrderLimitBy.h
View File

@ -8,7 +8,7 @@ namespace DB
class Context;
class WriteBuffer;
class TypeAndConstantInference;
struct TypeAndConstantInference;
/** Transform GROUP BY, ORDER BY and LIMIT BY sections.

10
dbms/include/DB/Analyzers/TypeAndConstantInference.h
View File

@ -11,8 +11,9 @@ namespace DB
class Context;
class WriteBuffer;
class CollectAliases;
class AnalyzeColumns;
struct CollectAliases;
struct AnalyzeColumns;
struct AnalyzeLambdas;
class IFunction;
class IAggregateFunction;
@ -29,7 +30,10 @@ class IAggregateFunction;
*/
struct TypeAndConstantInference
{
void process(ASTPtr & ast, Context & context, CollectAliases & aliases, const AnalyzeColumns & columns);
void process(ASTPtr & ast, Context & context,
CollectAliases & aliases,
const AnalyzeColumns & columns,
const AnalyzeLambdas & analyze_lambdas);
struct ExpressionInfo
{

77
dbms/include/DB/Columns/ColumnAggregateFunction.h
View File

@ -221,33 +221,18 @@ public:
return res;
}
void insertRangeFrom(const IColumn & from, size_t start, size_t length) override
size_t allocatedSize() const override
{
const ColumnAggregateFunction & from_concrete = static_cast<const ColumnAggregateFunction &>(from);
size_t res = getData().allocated_size() * sizeof(getData()[0]);
if (start + length > from_concrete.getData().size())
throw Exception("Parameters start = "
+ toString(start) + ", length = "
+ toString(length) + " are out of bound in ColumnAggregateFunction::insertRangeFrom method"
" (data.size() = " + toString(from_concrete.getData().size()) + ").",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
for (const auto & arena : arenas)
res += arena.get()->size();
if (src && src.get() != &from_concrete)
{
throw Exception("ColumnAggregateFunction could have only one source that owns aggregation states", ErrorCodes::BAD_ARGUMENTS);
}
else
{
/// Keep shared ownership of aggregation states.
src = from_concrete.shared_from_this();
}
auto & data = getData();
size_t old_size = data.size();
data.resize(old_size + length);
memcpy(&data[old_size], &from_concrete.getData()[start], length * sizeof(data[0]));
return res;
}
void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
void popBack(size_t n) override
{
size_t size = data.size();
@ -260,53 +245,9 @@ public:
data.resize_assume_reserved(new_size);
}
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override
{
size_t size = getData().size();
if (size != filter.size())
throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
std::shared_ptr<ColumnAggregateFunction> res = std::make_shared<ColumnAggregateFunction>(*this);
if (size == 0)
return res;
auto & res_data = res->getData();
if (result_size_hint)
res_data.reserve(result_size_hint > 0 ? result_size_hint : size);
for (size_t i = 0; i < size; ++i)
if (filter[i])
res_data.push_back(getData()[i]);
/// Для экономии оперативки в случае слишком сильной фильтрации.
if (res_data.size() * 2 < res_data.capacity())
res_data = Container_t(res_data.cbegin(), res_data.cend());
return res;
}
ColumnPtr permute(const Permutation & perm, size_t limit) const override
{
size_t size = getData().size();
if (limit == 0)
limit = size;
else
limit = std::min(size, limit);
if (perm.size() < limit)
throw Exception("Size of permutation is less than required.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
std::shared_ptr<ColumnAggregateFunction> res = std::make_shared<ColumnAggregateFunction>(*this);
res->getData().resize(limit);
for (size_t i = 0; i < limit; ++i)
res->getData()[i] = getData()[perm[i]];
return res;
}
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr replicate(const Offsets_t & offsets) const override
{

5
dbms/include/DB/Columns/ColumnArray.h
View File

@ -270,6 +270,11 @@ public:
return getData().byteSize() + getOffsets().size() * sizeof(getOffsets()[0]);
}
size_t allocatedSize() const override
{
return getData().allocatedSize() + getOffsets().allocated_size() * sizeof(getOffsets()[0]);
}
bool hasEqualOffsets(const ColumnArray & other) const
{
if (offsets == other.offsets)

1
dbms/include/DB/Columns/ColumnConst.h
View File

@ -159,6 +159,7 @@ public:
}
size_t byteSize() const override { return sizeof(data) + sizeof(s); }
size_t allocatedSize() const override { return byteSize(); }
ColumnPtr permute(const Permutation & perm, size_t limit) const override
{

5
dbms/include/DB/Columns/ColumnFixedString.h
View File

@ -84,6 +84,11 @@ public:
return chars.size() + sizeof(n);
}
size_t allocatedSize() const override
{
return chars.allocated_size() + sizeof(n);
}
Field operator[](size_t index) const override
{
return String(reinterpret_cast<const char *>(&chars[n * index]), n);

2
dbms/include/DB/Columns/ColumnNullable.h
View File

@ -39,6 +39,7 @@ public:
const char * deserializeAndInsertFromArena(const char * pos) override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;;
void insert(const Field & x) override;
void insertFrom(const IColumn & src, size_t n) override;
void insertDefault() override;
void popBack(size_t n) override;
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
@ -47,6 +48,7 @@ public:
void getPermutation(bool reverse, size_t limit, Permutation & res) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedSize() const override;
ColumnPtr replicate(const Offsets_t & replicate_offsets) const override;
ColumnPtr convertToFullColumnIfConst() const override;
void updateHashWithValue(size_t n, SipHash & hash) const override;

5
dbms/include/DB/Columns/ColumnSet.h
View File

@ -1,12 +1,15 @@
#pragma once
#include <DB/Columns/IColumnDummy.h>
#include <DB/Interpreters/Set.h>
namespace DB
{
class Set;
using ConstSetPtr = std::shared_ptr<const Set>;
/** Столбец, содержащий множество значений в секции IN.
* Ведёт себя как столбец-константа (так как множество одно, а не своё на каждую строку).
* Значение у этого столбца нестандартное, поэтому его невозможно получить через обычный интерфейс.

5
dbms/include/DB/Columns/ColumnString.h
View File

@ -63,6 +63,11 @@ public:
return chars.size() + offsets.size() * sizeof(offsets[0]);
}
size_t allocatedSize() const override
{
return chars.allocated_size() + offsets.allocated_size() * sizeof(offsets[0]);
}
ColumnPtr cloneResized(size_t size) const override
{
ColumnPtr new_col_holder = std::make_shared<ColumnString>();

8
dbms/include/DB/Columns/ColumnTuple.h
View File

@ -247,6 +247,14 @@ public:
return res;
}
size_t allocatedSize() const override
{
size_t res = 0;
for (const auto & column : columns)
res += column->allocatedSize();
return res;
}
ColumnPtr convertToFullColumnIfConst() const override
{
Block materialized = data;

5
dbms/include/DB/Columns/ColumnVector.h
View File

@ -214,6 +214,11 @@ public:
return data.size() * sizeof(data[0]);
}
size_t allocatedSize() const override
{
return data.allocated_size() * sizeof(data[0]);
}
void insert(const T value)
{
data.push_back(value);

8
dbms/include/DB/Columns/IColumn.h
View File

@ -259,9 +259,15 @@ public:
*/
virtual void reserve(size_t n) {};
/** Приблизительный размер столбца в оперативке в байтах - для профайлинга. 0 - если неизвестно. */
/** Size of column data in memory (may be approximate) - for profiling. Zero, if could not be determined. */
virtual size_t byteSize() const = 0;
/** Size of memory, allocated for column.
* This is greater or equals to byteSize due to memory reservation in containers.
* Zero, if could be determined.
*/
virtual size_t allocatedSize() const = 0;
virtual ~IColumn() {}
};

1
dbms/include/DB/Columns/IColumnDummy.h
View File

@ -30,6 +30,7 @@ public:
void insertDefault() override { ++s; }
void popBack(size_t n) override { s -= n; }
size_t byteSize() const override { return 0; }
size_t allocatedSize() const override { return 0; }
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override { return 0; }
Field operator[](size_t n) const override { throw Exception("Cannot get value from " + getName(), ErrorCodes::NOT_IMPLEMENTED); }

20
dbms/include/DB/Common/Collator.h
View File

@ -31,9 +31,9 @@ class Collator : private boost::noncopyable
{
public:
explicit Collator(const std::string & locale_) : locale(Poco::toLower(locale_))
{
{
UErrorCode status = U_ZERO_ERROR;
collator = ucol_open(locale.c_str(), &status);
if (status != U_ZERO_ERROR)
{
@ -41,38 +41,38 @@ public:
throw DB::Exception("Unsupported collation locale: " + locale, DB::ErrorCodes::UNSUPPORTED_COLLATION_LOCALE);
}
}
~Collator()
{
ucol_close(collator);
}
int compare(const char * str1, size_t length1, const char * str2, size_t length2) const
{
UCharIterator iter1, iter2;
uiter_setUTF8(&iter1, str1, length1);
uiter_setUTF8(&iter2, str2, length2);
UErrorCode status = U_ZERO_ERROR;
UCollationResult compare_result = ucol_strcollIter(collator, &iter1, &iter2, &status);
if (status != U_ZERO_ERROR)
throw DB::Exception("ICU collation comparison failed with error code: " + DB::toString(status),
DB::ErrorCodes::COLLATION_COMPARISON_FAILED);
/** Значения enum UCollationResult совпадают с нужными нам:
/** Values of enum UCollationResult are equals to what exactly we need:
* UCOL_EQUAL = 0
* UCOL_GREATER = 1
* UCOL_LESS = -1
*/
return compare_result;
}
const std::string & getLocale() const
{
return locale;
}
private:
std::string locale;
UCollator * collator;

50
dbms/include/DB/Common/memcpySmall.h
View File

@ -9,23 +9,25 @@
#include <emmintrin.h>
/** Функция memcpy может работать неоптимально при выполнении одновременно следующих условий:
* 1. Размер куска памяти сравнительно небольшой (как правило меньше, приблизительно, 50 байт).
* 2. Размер куска памяти неизвестен во время компиляции.
/** memcpy function could work suboptimal if all the following conditions are met:
* 1. Size of memory region is relatively small (approximately, under 50 bytes).
* 2. Size of memory region is not known at compile-time.
*
* В этом случае, она работает неоптимально по следующим причинам:
* 1. Функция не инлайнится.
* 2. Тратится много времени/инструкций на обработку "хвостиков" данных.
* In that case, memcpy works suboptimal by following reasons:
* 1. Function is not inlined.
* 2. Much time/instructions are spend to process "tails" of data.
*
* Существуют ситуации, когда функцию можно сделать быстрее, воспользовавшись некоторыми допущениями.
* Одно из таких допущений - возможность читать и писать какое-то количество байт после конца соответствующих диапазонов памяти.
* Тогда можно не тратить код на обработку хвостиков, а выполнять копирование всегда большими кусками.
* There are cases when function could be implemented in more optimal way, with help of some assumptions.
* One of that assumptions - ability to read and write some number of bytes after end of passed memory regions.
* Under that assumption, it is possible not to implement difficult code to process tails of data and do copy always by big chunks.
*
* Эта ситуация является типичной, например, когда короткие куски памяти копируются подряд в один непрервыный кусок памяти
* - так как каждое следующее копирование будет перетирать лишние данные от предыдущего копирования.
* This case is typical, for example, when many small pieces of data are gathered to single contiguous piece of memory in a loop.
* - because each next copy will overwrite excessive data after previous copy.
*
* Допущение о том, что размер небольшой, позволяет нам не разворачивать цикл.
* Это работает медленее, когда размер, на самом деле, большой.
* Assumption that size of memory region is small enough allows us to not unroll the loop.
* This is slower, when size of memory is actually big.
*
* Use with caution.
*/
namespace detail
@ -44,35 +46,23 @@ namespace detail
}
}
/** Исходит из допущения, что можно читать до 15 лишних байт после конца массива src,
* и записывать любой мусор до 15 байт после конца массива dst.
/** Works under assumption, that it's possible to read up to 15 excessive bytes after end of 'src' region
* and to write any garbage into up to 15 bytes after end of 'dst' region.
*/
inline void memcpySmallAllowReadWriteOverflow15(void * __restrict dst, const void * __restrict src, size_t n)
{
detail::memcpySmallAllowReadWriteOverflow15Impl(reinterpret_cast<char *>(dst), reinterpret_cast<const char *>(src), n);
}
/** Исходит из допущения, что можно записывать любой мусор до 15 байт после конца массива dst.
/** NOTE There was also a function, that assumes, that you could read any bytes inside same memory page of src.
* This function was unused, and also it requires special handling for Valgrind and ASan.
*/
inline void memcpySmallAllowWriteOverflow15(void * __restrict dst, const void * __restrict src, size_t n)
{
if (reinterpret_cast<intptr_t>(src) % 4096 <= 4096 - 16)
memcpySmallAllowReadWriteOverflow15(dst, src, n);
else
memcpy(dst, src, n);
}
#else /// Реализации для других платформ.
#else /// Implementation for other platforms.
inline void memcpySmallAllowReadWriteOverflow15(void * __restrict dst, const void * __restrict src, size_t n)
{
memcpy(dst, src, n);
}
inline void memcpySmallAllowWriteOverflow15(void * __restrict dst, const void * __restrict src, size_t n)
{
memcpy(dst, src, n);
}
#endif

222
dbms/include/DB/Core/Field.h
View File

@ -44,7 +44,7 @@ class Field
public:
struct Types
{
/// Идентификатор типа.
/// Type tag.
enum Which
{
Null = 0,
@ -52,7 +52,7 @@ public:
Int64 = 2,
Float64 = 3,
/// не POD типы. Для них предполагается relocatable.
/// Non-POD types.
String = 16,
Array = 17,
@ -79,6 +79,7 @@ public:
}
};
/// Позволяет получить идентификатор для типа или наоборот.
template <typename T> struct TypeToEnum;
template <Types::Which which> struct EnumToType;
@ -99,13 +100,14 @@ public:
Field(Field && rhs)
{
move(std::move(rhs));
create(std::move(rhs));
}
template <typename T>
Field(const T & rhs)
Field(T && rhs,
typename std::enable_if<!std::is_same<typename std::decay<T>::type, Field>::value, void>::type * unused = nullptr)
{
create(rhs);
createConcrete(std::forward<T>(rhs));
}
/// Создать строку inplace.
@ -119,6 +121,7 @@ public:
create(data, size);
}
/// NOTE In case when field already has string type, more direct assign is possible.
void assignString(const char * data, size_t size)
{
destroy();
@ -135,8 +138,13 @@ public:
{
if (this != &rhs)
{
destroy();
create(rhs);
if (which != rhs.which)
{
destroy();
create(rhs);
}
else
assign(rhs); /// This assigns string or vector without deallocation of existing buffer.
}
return *this;
}
@ -145,16 +153,29 @@ public:
{
if (this != &rhs)
{
move(std::move(rhs));
if (which != rhs.which)
{
destroy();
create(std::move(rhs));
}
else
assign(std::move(rhs));
}
return *this;
}
template <typename T>
Field & operator= (const T & rhs)
typename std::enable_if<!std::is_same<typename std::decay<T>::type, Field>::value, Field &>::type
operator= (T && rhs)
{
destroy();
create(rhs);
if (which != TypeToEnum<typename std::decay<T>::type>::value)
{
destroy();
createConcrete(std::forward<T>(rhs));
}
else
assignConcrete(std::forward<T>(rhs));
return *this;
}
@ -186,7 +207,7 @@ public:
template <typename T> T & safeGet()
{
const Types::Which requested = TypeToEnum<typename std::remove_cv<typename std::remove_reference<T>::type>::type>::value;
const Types::Which requested = TypeToEnum<typename std::decay<T>::type>::value;
if (which != requested)
throw Exception("Bad get: has " + std::string(getTypeName()) + ", requested " + std::string(Types::toString(requested)), ErrorCodes::BAD_GET);
return get<T>();
@ -194,7 +215,7 @@ public:
template <typename T> const T & safeGet() const
{
const Types::Which requested = TypeToEnum<typename std::remove_cv<typename std::remove_reference<T>::type>::type>::value;
const Types::Which requested = TypeToEnum<typename std::decay<T>::type>::value;
if (which != requested)
throw Exception("Bad get: has " + std::string(getTypeName()) + ", requested " + std::string(Types::toString(requested)), ErrorCodes::BAD_GET);
return get<T>();
@ -289,38 +310,71 @@ private:
Types::Which which;
/// Assuming there was no allocated state or it was deallocated (see destroy).
template <typename T>
void create(const T & x)
void createConcrete(T && x)
{
which = TypeToEnum<T>::value;
T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(storage);
new (ptr) T(x);
using JustT = typename std::decay<T>::type;
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(storage);
new (ptr) JustT(std::forward<T>(x));
which = TypeToEnum<JustT>::value;
}
void create(const Null & x)
/// Assuming same types.
template <typename T>
void assignConcrete(T && x)
{
which = Types::Null;
using JustT = typename std::decay<T>::type;
JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast<JustT *>(storage);
*ptr = std::forward<T>(x);
}
void create(const Field & x)
template <typename F, typename Field> /// Field template parameter may be const or non-const Field.
static void dispatch(F && f, Field & field)
{
switch (x.which)
switch (field.which)
{
case Types::Null: create(Null()); break;
case Types::UInt64: create(x.get<UInt64>()); break;
case Types::Int64: create(x.get<Int64>()); break;
case Types::Float64: create(x.get<Float64>()); break;
case Types::String: create(x.get<String>()); break;
case Types::Array: create(x.get<Array>()); break;
case Types::Tuple: create(x.get<Tuple>()); break;
case Types::Null: f(field.template get<Null>()); return;
case Types::UInt64: f(field.template get<UInt64>()); return;
case Types::Int64: f(field.template get<Int64>()); return;
case Types::Float64: f(field.template get<Float64>()); return;
case Types::String: f(field.template get<String>()); return;
case Types::Array: f(field.template get<Array>()); return;
case Types::Tuple: f(field.template get<Tuple>()); return;
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
}
void create(const Field & x)
{
dispatch([this] (auto & value) { createConcrete(value); }, x);
}
void create(Field && x)
{
dispatch([this] (auto & value) { createConcrete(std::move(value)); }, x);
}
void assign(const Field & x)
{
dispatch([this] (auto & value) { assignConcrete(value); }, x);
}
void assign(Field && x)
{
dispatch([this] (auto & value) { assignConcrete(std::move(value)); }, x);
}
void create(const char * data, size_t size)
{
which = Types::String;
String * __attribute__((__may_alias__)) ptr = reinterpret_cast<String*>(storage);
new (ptr) String(data, size);
which = Types::String;
}
void create(const unsigned char * data, size_t size)
@ -347,6 +401,8 @@ private:
default:
break;
}
which = Types::Null; /// for exception safety in subsequent calls to destroy and create, when create fails.
}
template <typename T>
@ -355,31 +411,6 @@ private:
T * __attribute__((__may_alias__)) ptr = reinterpret_cast<T*>(storage);
ptr->~T();
}
template <typename T>
void moveValue(Field && x)
{
T * __attribute__((__may_alias__)) ptr_this = reinterpret_cast<T*>(storage);
T * __attribute__((__may_alias__)) ptr_x = reinterpret_cast<T*>(x.storage);
new (ptr_this) T(std::move(*ptr_x));
}
void move(Field && x)
{
which = x.which;
switch (x.which)
{
case Types::Null: create(Null()); break;
case Types::UInt64: create(x.get<UInt64>()); break;
case Types::Int64: create(x.get<Int64>()); break;
case Types::Float64: create(x.get<Float64>()); break;
case Types::String: moveValue<String>(std::move(x)); break;
case Types::Array: moveValue<Array>(std::move(x)); break;
case Types::Tuple: moveValue<Tuple>(std::move(x)); break;
}
}
};
#undef DBMS_MIN_FIELD_SIZE
@ -393,7 +424,7 @@ template <> struct Field::TypeToEnum<String> { static const Types::Which
template <> struct Field::TypeToEnum<Array> { static const Types::Which value = Types::Array; };
template <> struct Field::TypeToEnum<Tuple> { static const Types::Which value = Types::Tuple; };
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null ; };
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null ; };
template <> struct Field::EnumToType<Field::Types::UInt64> { using Type = UInt64 ; };
template <> struct Field::EnumToType<Field::Types::Int64> { using Type = Int64 ; };
template <> struct Field::EnumToType<Field::Types::Float64> { using Type = Float64 ; };
@ -447,7 +478,7 @@ template <> struct NearestFieldType<String> { using Type = String ; };
template <> struct NearestFieldType<Array> { using Type = Array ; };
template <> struct NearestFieldType<Tuple> { using Type = Tuple ; };
template <> struct NearestFieldType<bool> { using Type = UInt64 ; };
template <> struct NearestFieldType<Null> { using Type = Null; };
template <> struct NearestFieldType<Null> { using Type = Null; };
template <typename T>
@ -456,55 +487,32 @@ typename NearestFieldType<T>::Type nearestFieldType(const T & x)
return typename NearestFieldType<T>::Type(x);
}
}
/// Заглушки, чтобы DBObject-ы с полем типа Array компилировались.
#include <mysqlxx/Manip.h>
namespace mysqlxx
{
std::ostream & operator<< (mysqlxx::EscapeManipResult res, const DB::Array & value);
std::ostream & operator<< (mysqlxx::QuoteManipResult res, const DB::Array & value);
std::istream & operator>> (mysqlxx::UnEscapeManipResult res, DB::Array & value);
std::istream & operator>> (mysqlxx::UnQuoteManipResult res, DB::Array & value);
std::ostream & operator<< (mysqlxx::EscapeManipResult res, const DB::Tuple & value);
std::ostream & operator<< (mysqlxx::QuoteManipResult res, const DB::Tuple & value);
std::istream & operator>> (mysqlxx::UnEscapeManipResult res, DB::Tuple & value);
std::istream & operator>> (mysqlxx::UnQuoteManipResult res, DB::Tuple & value);
}
namespace DB
{
class ReadBuffer;
class WriteBuffer;
/// Предполагается что у всех элементов массива одинаковый тип.
void readBinary(Array & x, ReadBuffer & buf);
inline void readText(Array & x, ReadBuffer & buf) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); }
inline void readQuoted(Array & x, ReadBuffer & buf) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); }
/// Предполагается что у всех элементов массива одинаковый тип.
void writeBinary(const Array & x, WriteBuffer & buf);
void writeText(const Array & x, WriteBuffer & buf);
inline void writeQuoted(const Array & x, WriteBuffer & buf) { throw Exception("Cannot write Array quoted.", ErrorCodes::NOT_IMPLEMENTED); }
}
namespace DB
{
void readBinary(Tuple & x, ReadBuffer & buf);
inline void readText(Tuple & x, ReadBuffer & buf) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); }
inline void readQuoted(Tuple & x, ReadBuffer & buf) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); }
void writeBinary(const Tuple & x, WriteBuffer & buf);
void writeText(const Tuple & x, WriteBuffer & buf);
inline void writeQuoted(const Tuple & x, WriteBuffer & buf) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); }
class ReadBuffer;
class WriteBuffer;
/// Предполагается что у всех элементов массива одинаковый тип.
void readBinary(Array & x, ReadBuffer & buf);
inline void readText(Array & x, ReadBuffer & buf) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); }
inline void readQuoted(Array & x, ReadBuffer & buf) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); }
/// Предполагается что у всех элементов массива одинаковый тип.
void writeBinary(const Array & x, WriteBuffer & buf);
void writeText(const Array & x, WriteBuffer & buf);
inline void writeQuoted(const Array & x, WriteBuffer & buf) { throw Exception("Cannot write Array quoted.", ErrorCodes::NOT_IMPLEMENTED); }
void readBinary(Tuple & x, ReadBuffer & buf);
inline void readText(Tuple & x, ReadBuffer & buf) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); }
inline void readQuoted(Tuple & x, ReadBuffer & buf) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); }
void writeBinary(const Tuple & x, WriteBuffer & buf);
void writeText(const Tuple & x, WriteBuffer & buf);
inline void writeQuoted(const Tuple & x, WriteBuffer & buf) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); }
}

1
dbms/include/DB/Core/FieldVisitors.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <DB/Core/Field.h>
#include <common/DateLUT.h>
class SipHash;

38
dbms/include/DB/Core/Progress.h
View File

@ -1,15 +1,17 @@
#pragma once
#include <atomic>
#include <common/Common.h>
#include <DB/Core/Defines.h>
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
namespace DB
{
class ReadBuffer;
class WriteBuffer;
/** Progress of query execution.
* Values, transferred over network are deltas - how much was done after previously sent value.
@ -30,31 +32,11 @@ struct Progress
Progress(size_t rows_, size_t bytes_, size_t total_rows_ = 0)
: rows(rows_), bytes(bytes_), total_rows(total_rows_) {}
void read(ReadBuffer & in, UInt64 server_revision)
{
size_t new_rows = 0;
size_t new_bytes = 0;
size_t new_total_rows = 0;
void read(ReadBuffer & in, UInt64 server_revision);
void write(WriteBuffer & out, UInt64 client_revision) const;
readVarUInt(new_rows, in);
readVarUInt(new_bytes, in);
if (server_revision >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS)
readVarUInt(new_total_rows, in);
rows = new_rows;
bytes = new_bytes;
total_rows = new_total_rows;
}
void write(WriteBuffer & out, UInt64 client_revision) const
{
writeVarUInt(rows, out);
writeVarUInt(bytes, out);
if (client_revision >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS)
writeVarUInt(total_rows, out);
}
/// Progress in JSON format (single line, without whitespaces) is used in HTTP headers.
void writeJSON(WriteBuffer & out) const;
/// Each value separately is changed atomically (but not whole object).
void incrementPiecewiseAtomically(const Progress & rhs)

16
dbms/include/DB/Core/StringRef.h
View File

@ -1,11 +1,7 @@
#pragma once
#include <string.h>
#include <city.h>
#include <string>
#include <vector>
#include <functional>
#include <ostream>
@ -13,6 +9,8 @@
#include <emmintrin.h>
#endif
#include <city.h>
#include <DB/Core/Types.h>
#include <DB/Common/unaligned.h>
@ -215,11 +213,11 @@ inline size_t hashLessThan8(const char * data, size_t size)
if (size > 0)
{
uint8 a = data[0];
uint8 b = data[size >> 1];
uint8 c = data[size - 1];
uint32 y = static_cast<uint32>(a) + (static_cast<uint32>(b) << 8);
uint32 z = size + (static_cast<uint32>(c) << 2);
uint8_t a = data[0];
uint8_t b = data[size >> 1];
uint8_t c = data[size - 1];
uint32_t y = static_cast<uint32_t>(a) + (static_cast<uint32_t>(b) << 8);
uint32_t z = size + (static_cast<uint32_t>(c) << 2);
return shiftMix(y * k2 ^ z * k3) * k2;
}

6
dbms/include/DB/DataStreams/ColumnGathererStream.h
View File

@ -1,10 +1,12 @@
#pragma once
#include <DB/DataStreams/IProfilingBlockInputStream.h>
#include <DB/Storages/IStorage.h>
#include <DB/Common/PODArray.h>
namespace Poco { class Logger; }
namespace DB
{
@ -102,7 +104,7 @@ private:
size_t pos_global_start = 0;
size_t block_preferred_size;
Logger * log = &Logger::get("ColumnGathererStream");
Poco::Logger * log;
};
}

18
dbms/include/DB/DataStreams/IBlockInputStream.h
View File

@ -1,10 +1,10 @@
#pragma once
#include <vector>
#include <memory>
#include <functional>
#include <boost/noncopyable.hpp>
#include <DB/Core/Block.h>
#include <DB/Core/Progress.h>
#include <DB/Storages/IStorage.h>
namespace DB
@ -16,6 +16,14 @@ class IBlockInputStream;
using BlockInputStreamPtr = std::shared_ptr<IBlockInputStream>;
using BlockInputStreams = std::vector<BlockInputStreamPtr>;
class TableStructureReadLock;
using TableStructureReadLockPtr = std::shared_ptr<TableStructureReadLock>;
using TableStructureReadLocks = std::vector<TableStructureReadLockPtr>;
struct Progress;
/** Коллбэк для отслеживания прогресса выполнения запроса.
* Используется в IProfilingBlockInputStream и Context-е.
@ -84,10 +92,10 @@ public:
/** Не давать изменить таблицу, пока жив поток блоков.
*/
void addTableLock(const IStorage::TableStructureReadLockPtr & lock) { table_locks.push_back(lock); }
void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); }
protected:
IStorage::TableStructureReadLocks table_locks;
TableStructureReadLocks table_locks;
BlockInputStreams children;

16
dbms/include/DB/DataStreams/IBlockOutputStream.h
View File

@ -1,7 +1,9 @@
#pragma once
#include <string>
#include <vector>
#include <memory>
#include <boost/noncopyable.hpp>
#include <DB/Storages/IStorage.h>
namespace DB
@ -10,6 +12,12 @@ namespace DB
class Block;
struct Progress;
class TableStructureReadLock;
using TableStructureReadLockPtr = std::shared_ptr<TableStructureReadLock>;
using TableStructureReadLocks = std::vector<TableStructureReadLockPtr>;
struct Progress;
/** Interface of stream for writing data (into table, filesystem, network, terminal, etc.)
*/
@ -44,16 +52,16 @@ public:
/** Content-Type to set when sending HTTP response.
*/
virtual String getContentType() const { return "text/plain; charset=UTF-8"; }
virtual std::string getContentType() const { return "text/plain; charset=UTF-8"; }
virtual ~IBlockOutputStream() {}
/** Don't let to alter table while instance of stream is alive.
*/
void addTableLock(const IStorage::TableStructureReadLockPtr & lock) { table_locks.push_back(lock); }
void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); }
protected:
IStorage::TableStructureReadLocks table_locks;
TableStructureReadLocks table_locks;
};
using BlockOutputStreamPtr = std::shared_ptr<IBlockOutputStream>;

34
dbms/include/DB/DataStreams/MaterializingBlockOutputStream.h
View File

@ -1,10 +1,8 @@
#pragma once
#include <DB/Columns/ColumnConst.h>
#include <DB/Core/Block.h>
#include <DB/DataStreams/IBlockOutputStream.h>
#include <DB/DataTypes/DataTypeNullable.h>
#include <DB/DataTypes/DataTypesNumberFixed.h>
#include <ext/range.hpp>
namespace DB
{
@ -19,7 +17,7 @@ public:
void write(const Block & block) override
{
output->write(materialize(block));
output->write(materialize(block));
}
void flush() override { output->flush(); }
@ -34,31 +32,7 @@ public:
private:
BlockOutputStreamPtr output;
static Block materialize(const Block & original_block)
{
/// copy block to get rid of const
auto block = original_block;
for (const auto i : ext::range(0, block.columns()))
{
auto & element = block.safeGetByPosition(i);
auto & src = element.column;
ColumnPtr converted = src->convertToFullColumnIfConst();
if (converted)
{
src = converted;
auto & type = element.type;
if (type->isNull())
{
/// A ColumnNull that is converted to a full column
/// has the type DataTypeNullable(DataTypeUInt8).
type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt8>());
}
}
}
return block;
}
static Block materialize(const Block & original_block);
};
}

4
dbms/include/DB/DataStreams/NullAndDoCopyBlockInputStream.h
View File

@ -7,6 +7,10 @@
namespace DB
{
class IBlockOutputStream;
using BlockOutputStreamPtr = std::shared_ptr<IBlockOutputStream>;
/** Пустой поток блоков.
* Но при первой попытке чтения, копирует данные из переданного input-а в переданный output.
* Это нужно для выполнения запроса INSERT SELECT - запрос копирует данные, но сам ничего не возвращает.

3
dbms/include/DB/DataStreams/ODBCDriverBlockOutputStream.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <string>
#include <DB/DataStreams/IBlockOutputStream.h>
@ -23,7 +24,7 @@ public:
void write(const Block & block) override;
void flush() override;
String getContentType() const override { return "application/octet-stream"; }
std::string getContentType() const override { return "application/octet-stream"; }
private:
bool is_first = true;

1
dbms/include/DB/DataStreams/PrettyBlockOutputStream.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <DB/Core/Block.h>
#include <DB/DataStreams/IBlockOutputStream.h>

10
dbms/include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <DB/DataStreams/IBlockOutputStream.h>
#include <DB/Core/NamesAndTypes.h>
namespace DB
@ -22,14 +23,7 @@ public:
}
private:
void write(const Block & block) override
{
for (const auto & column : columns)
if (block.has(column.name))
throw Exception{"Cannot insert column " + column.name, ErrorCodes::ILLEGAL_COLUMN};
output->write(block);
}
void write(const Block & block) override;
void flush() override { output->flush(); }

6
dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h
View File

@ -1,14 +1,8 @@
#pragma once
#include <common/logger_useful.h>
#include <DB/Core/FieldVisitors.h>
#include <DB/Core/Row.h>
#include <DB/Core/ColumnNumbers.h>
#include <DB/DataStreams/MergingSortedBlockInputStream.h>
#include <DB/Storages/MergeTree/PKCondition.h>
#include <ext/range.hpp>
#include <ext/map.hpp>
namespace DB

25
dbms/include/DB/DataTypes/DataTypeExpression.h
View File

@ -6,8 +6,7 @@
namespace DB
{
/**
* Лямбда-выражение.
/** Special data type, representing lambda expression.
*/
class DataTypeExpression final : public IDataTypeDummy
{
@ -16,29 +15,11 @@ private:
DataTypePtr return_type;
public:
/// Некоторые типы могут быть еще не известны.
/// Some types could be still unknown.
DataTypeExpression(DataTypes argument_types_ = DataTypes(), DataTypePtr return_type_ = nullptr)
: argument_types(argument_types_), return_type(return_type_) {}
std::string getName() const override
{
std::string res = "Expression(";
if (argument_types.size() > 1)
res += "(";
for (size_t i = 0; i < argument_types.size(); ++i)
{
if (i > 0)
res += ", ";
const DataTypePtr & type = argument_types[i];
res += type ? type->getName() : "?";
}
if (argument_types.size() > 1)
res += ")";
res += " -> ";
res += return_type ? return_type->getName() : "?";
res += ")";
return res;
}
std::string getName() const override;
DataTypePtr clone() const override
{

7
dbms/include/DB/Databases/DatabaseCloud.h
View File

@ -92,13 +92,16 @@ public:
bool empty() const override;
void createTable(const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine) override;
void createTable(
const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine, const Settings & settings) override;
void removeTable(const String & table_name) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
void renameTable(const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name) override;
void renameTable(
const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name, const Settings & settings) override;
time_t getTableMetadataModificationTime(const String & name) override;

13
dbms/include/DB/Databases/DatabaseMemory.h
View File

@ -1,9 +1,13 @@
#pragma once
#include <mutex>
#include <DB/Databases/IDatabase.h>
#include <DB/Storages/IStorage.h>
namespace Poco { class Logger; }
namespace DB
{
@ -19,7 +23,7 @@ protected:
mutable std::mutex mutex;
Tables tables;
Logger * log;
Poco::Logger * log;
public:
@ -36,13 +40,16 @@ public:
bool empty() const override;
void createTable(const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine) override;
void createTable(
const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine, const Settings & settings) override;
void removeTable(const String & table_name) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
void renameTable(const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name) override;
void renameTable(
const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name, const Settings & settings) override;
time_t getTableMetadataModificationTime(const String & table_name) override;

7
dbms/include/DB/Databases/DatabaseOrdinary.h
View File

@ -22,10 +22,13 @@ public:
void loadTables(Context & context, ThreadPool * thread_pool, bool has_force_restore_data_flag) override;
void createTable(const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine) override;
void createTable(
const String & table_name, const StoragePtr & table, const ASTPtr & query, const String & engine, const Settings & settings) override;
void removeTable(const String & table_name) override;
void renameTable(const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name) override;
void renameTable(
const Context & context, const String & table_name, IDatabase & to_database, const String & to_table_name, const Settings & settings) override;
time_t getTableMetadataModificationTime(const String & table_name) override;

26
dbms/include/DB/Databases/IDatabase.h
View File

@ -22,8 +22,10 @@ using StoragePtr = std::shared_ptr<IStorage>;
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
struct Settings;
/** Позволяет проитерироваться по списку таблиц.
/** Allows to iterate over tables.
*/
class IDatabaseIterator
{
@ -40,19 +42,19 @@ public:
using DatabaseIteratorPtr = std::unique_ptr<IDatabaseIterator>;
/** Движок баз данных.
* Отвечает за:
* - инициализацию множества таблиц;
* - проверку существования и получение таблицы для работы;
* - получение списка всех таблиц;
* - создание и удаление таблиц;
* - переименовывание таблиц и перенос между БД с одинаковыми движками.
/** Database engine.
* It is responsible for:
* - initialization of set of known tables;
* - checking existence of a table and getting a table object;
* - retrieving a list of all tables;
* - creating and dropping tables;
* - renaming tables and moving between databases with same engine.
*/
class IDatabase : public std::enable_shared_from_this<IDatabase>
{
public:
/// Получить имя движка базы данных.
/// Get name of database engine.
virtual String getEngineName() const = 0;
/// Загрузить множество существующих таблиц. Если задан thread_pool - использовать его.
@ -73,7 +75,8 @@ public:
virtual bool empty() const = 0;
/// Добавить таблицу в базу данных. Прописать её наличие в метаданных.
virtual void createTable(const String & name, const StoragePtr & table, const ASTPtr & query, const String & engine) = 0;
virtual void createTable(
const String & name, const StoragePtr & table, const ASTPtr & query, const String & engine, const Settings & settings) = 0;
/// Удалить таблицу из базы данных и вернуть её. Удалить метаданные.
virtual void removeTable(const String & name) = 0;
@ -85,7 +88,8 @@ public:
virtual StoragePtr detachTable(const String & name) = 0;
/// Переименовать таблицу и, возможно, переместить таблицу в другую БД.
virtual void renameTable(const Context & context, const String & name, IDatabase & to_database, const String & to_name) = 0;
virtual void renameTable(
const Context & context, const String & name, IDatabase & to_database, const String & to_name, const Settings & settings) = 0;
/// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
virtual time_t getTableMetadataModificationTime(const String & name) = 0;

2
dbms/include/DB/Dictionaries/DictionaryFactory.h
View File

@ -2,6 +2,8 @@
#include <DB/Dictionaries/IDictionary.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <common/singleton.h>
namespace DB
{

43
dbms/include/DB/Dictionaries/Embedded/RegionsHierarchies.h
View File

@ -1,7 +1,8 @@
#pragma once
#include <DB/Dictionaries/Embedded/RegionsHierarchy.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/Exception.h>
#include <unordered_map>
/** Содержит несколько иерархий регионов, загружаемых из нескольких разных файлов.
@ -13,50 +14,20 @@ class RegionsHierarchies
private:
using Container = std::unordered_map<std::string, RegionsHierarchy>;
Container data;
Logger * log = &Logger::get("RegionsHierarchies");
public:
static constexpr auto required_key = "path_to_regions_hierarchy_file";
/** path должен указывать на файл с иерархией регионов "по-умолчанию". Она будет доступна по пустому ключу.
/** path_to_regions_hierarchy_file in configuration file
* должен указывать на файл с иерархией регионов "по-умолчанию". Она будет доступна по пустому ключу.
* Кроме того, рядом ищутся файлы, к имени которых (до расширения, если есть) добавлен произвольный _suffix.
* Такие файлы загружаются, и иерархия регионов кладётся по ключу suffix.
*
* Например, если указано /opt/geo/regions_hierarchy.txt,
* то будет также загружен файл /opt/geo/regions_hierarchy_ua.txt, если такой есть - он будет доступен по ключу ua.
*/
RegionsHierarchies(const std::string & default_path = Poco::Util::Application::instance().config().getString(required_key))
{
LOG_DEBUG(log, "Adding default regions hierarchy from " << default_path);
RegionsHierarchies();
data.emplace(std::piecewise_construct,
std::forward_as_tuple(""),
std::forward_as_tuple(default_path));
std::string basename = Poco::Path(default_path).getBaseName();
Poco::Path dir_path = Poco::Path(default_path).absolute().parent();
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(dir_path); dir_it != dir_end; ++dir_it)
{
std::string other_basename = dir_it.path().getBaseName();
if (0 == other_basename.compare(0, basename.size(), basename) && other_basename.size() > basename.size() + 1)
{
if (other_basename[basename.size()] != '_')
continue;
std::string suffix = other_basename.substr(basename.size() + 1);
LOG_DEBUG(log, "Adding regions hierarchy from " << dir_it->path() << ", key: " << suffix);
data.emplace(std::piecewise_construct,
std::forward_as_tuple(suffix),
std::forward_as_tuple(dir_it->path()));
}
}
}
/// Has corresponding section in configuration file.
static bool isConfigured();
/** Перезагружает, при необходимости, все иерархии регионов.

196
dbms/include/DB/Dictionaries/Embedded/RegionsHierarchy.h
View File

@ -1,17 +1,10 @@
#pragma once
#include <Poco/Util/Application.h>
#include <Poco/Exception.h>
#include <Poco/File.h>
#include <common/logger_useful.h>
#include <vector>
#include <boost/noncopyable.hpp>
#include <common/Common.h>
#include <common/singleton.h>
#include <DB/IO/ReadBufferFromFile.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <boost/noncopyable.hpp>
#define REGION_TYPE_CITY 6
#define REGION_TYPE_AREA 5
@ -27,9 +20,7 @@
class RegionsHierarchy : private boost::noncopyable
{
private:
std::string path;
time_t file_modification_time;
Logger * log;
time_t file_modification_time = 0;
using RegionID = UInt32;
using RegionType = UInt8;
@ -66,182 +57,15 @@ private:
/// регион - глубина в дереве
RegionDepths depths;
public:
RegionsHierarchy(const std::string & path_ = Poco::Util::Application::instance().config().getString("path_to_regions_hierarchy_file"))
: path(path_), file_modification_time(0), log(&Logger::get("RegionsHierarchy"))
{
}
/// path to file with data
std::string path;
public:
RegionsHierarchy();
RegionsHierarchy(const std::string & path_);
/// Перезагружает, при необходимости, иерархию регионов. Непотокобезопасно.
void reload()
{
time_t new_modification_time = Poco::File(path).getLastModified().epochTime();
if (new_modification_time <= file_modification_time)
return;
file_modification_time = new_modification_time;
LOG_DEBUG(log, "Reloading regions hierarchy");
const size_t initial_size = 10000;
const size_t max_size = 1000000;
RegionParents new_parents(initial_size);
RegionParents new_city(initial_size);
RegionParents new_country(initial_size);
RegionParents new_area(initial_size);
RegionParents new_district(initial_size);
RegionParents new_continent(initial_size);
RegionParents new_top_continent(initial_size);
RegionPopulations new_populations(initial_size);
RegionDepths new_depths(initial_size);
RegionTypes types(initial_size);
DB::ReadBufferFromFile in(path);
RegionID max_region_id = 0;
while (!in.eof())
{
/** Our internal geobase has negative numbers,
* that means "this is garbage, ignore this row".
*/
Int32 read_region_id = 0;
Int32 read_parent_id = 0;
Int8 read_type = 0;
DB::readIntText(read_region_id, in);
DB::assertChar('\t', in);
DB::readIntText(read_parent_id, in);
DB::assertChar('\t', in);
DB::readIntText(read_type, in);
/** Далее может быть перевод строки (старый вариант)
* или таб, население региона, перевод строки (новый вариант).
*/
RegionPopulation population = 0;
if (!in.eof() && *in.position() == '\t')
{
++in.position();
UInt64 population_big = 0;
DB::readIntText(population_big, in);
population = population_big > std::numeric_limits<RegionPopulation>::max()
? std::numeric_limits<RegionPopulation>::max()
: population_big;
}
DB::assertChar('\n', in);
if (read_region_id <= 0 || read_type < 0)
continue;
RegionID region_id = read_region_id;
RegionID parent_id = 0;
if (read_parent_id >= 0)
parent_id = read_parent_id;
RegionType type = read_type;
if (region_id > max_region_id)
{
if (region_id > max_size)
throw DB::Exception("Region id is too large: " + DB::toString(region_id) + ", should be not more than " + DB::toString(max_size));
max_region_id = region_id;
while (region_id >= new_parents.size())
{
new_parents.resize(new_parents.size() * 2);
new_populations.resize(new_parents.size());
types.resize(new_parents.size());
}
}
new_parents[region_id] = parent_id;
new_populations[region_id] = population;
types[region_id] = type;
}
new_parents .resize(max_region_id + 1);
new_city .resize(max_region_id + 1);
new_country .resize(max_region_id + 1);
new_area .resize(max_region_id + 1);
new_district .resize(max_region_id + 1);
new_continent .resize(max_region_id + 1);
new_top_continent.resize(max_region_id + 1);
new_populations .resize(max_region_id + 1);
new_depths .resize(max_region_id + 1);
types .resize(max_region_id + 1);
/// пропишем города и страны для регионов
for (RegionID i = 0; i <= max_region_id; ++i)
{
if (types[i] == REGION_TYPE_CITY)
new_city[i] = i;
if (types[i] == REGION_TYPE_AREA)
new_area[i] = i;
if (types[i] == REGION_TYPE_DISTRICT)
new_district[i] = i;
if (types[i] == REGION_TYPE_COUNTRY)
new_country[i] = i;
if (types[i] == REGION_TYPE_CONTINENT)
{
new_continent[i] = i;
new_top_continent[i] = i;
}
RegionDepth depth = 0;
RegionID current = i;
while (true)
{
++depth;
if (depth == std::numeric_limits<RegionDepth>::max())
throw Poco::Exception("Logical error in regions hierarchy: region " + DB::toString(current) + " possible is inside infinite loop");
current = new_parents[current];
if (current == 0)
break;
if (current > max_region_id)
throw Poco::Exception("Logical error in regions hierarchy: region " + DB::toString(current) + " (specified as parent) doesn't exist");
if (types[current] == REGION_TYPE_CITY)
new_city[i] = current;
if (types[current] == REGION_TYPE_AREA)
new_area[i] = current;
if (types[current] == REGION_TYPE_DISTRICT)
new_district[i] = current;
if (types[current] == REGION_TYPE_COUNTRY)
new_country[i] = current;
if (types[current] == REGION_TYPE_CONTINENT)
{
if (!new_continent[i])
new_continent[i] = current;
new_top_continent[i] = current;
}
}
new_depths[i] = depth;
}
parents.swap(new_parents);
country.swap(new_country);
city.swap(new_city);
area.swap(new_area);
district.swap(new_district);
continent.swap(new_continent);
top_continent.swap(new_top_continent);
populations.swap(new_populations);
depths.swap(new_depths);
}
void reload();
bool in(RegionID lhs, RegionID rhs) const

118
dbms/include/DB/Dictionaries/Embedded/RegionsNames.h
View File

@ -1,23 +1,13 @@
#pragma once
#include <sparsehash/dense_hash_map>
#include <Poco/File.h>
#include <Poco/NumberParser.h>
#include <Poco/Util/Application.h>
#include <string>
#include <vector>
#include <Poco/Exception.h>
#include <common/Common.h>
#include <common/logger_useful.h>
#include <DB/Core/StringRef.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromFile.h>
/** @brief Класс, позволяющий узнавать по id региона его текстовое название на одном из поддерживаемых языков: ru, en, ua, by, kz, tr.
/** Класс, позволяющий узнавать по id региона его текстовое название на одном из поддерживаемых языков: ru, en, ua, by, kz, tr.
*
* Информацию об именах регионов загружает из текстовых файлов с названиями следующего формата:
* regions_names_xx.txt,
@ -75,89 +65,13 @@ private:
using StringRefsForLanguageID = std::vector<StringRefs>;
public:
static constexpr auto required_key = "path_to_regions_names_files";
RegionsNames(const std::string & directory_ = Poco::Util::Application::instance().config().getString(required_key))
: directory(directory_)
{
}
/** @brief Перезагружает, при необходимости, имена регионов.
/** Перезагружает, при необходимости, имена регионов.
*/
void reload()
{
LOG_DEBUG(log, "Reloading regions names");
void reload();
RegionID max_region_id = 0;
for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id)
{
const std::string & language = getSupportedLanguages()[language_id];
std::string path = directory + "/regions_names_" + language + ".txt";
/// Has corresponding section in configuration file.
static bool isConfigured();
Poco::File file(path);
time_t new_modification_time = file.getLastModified().epochTime();
if (new_modification_time <= file_modification_times[language_id])
continue;
file_modification_times[language_id] = new_modification_time;
LOG_DEBUG(log, "Reloading regions names for language: " << language);
DB::ReadBufferFromFile in(path);
const size_t initial_size = 10000;
const size_t max_size = 1000000;
Chars new_chars;
StringRefs new_names_refs(initial_size, StringRef("", 0));
/// Выделим непрерывный кусок памяти, которого хватит для хранения всех имён.
new_chars.reserve(Poco::File(path).getSize());
while (!in.eof())
{
Int32 read_region_id;
std::string region_name;
DB::readIntText(read_region_id, in);
DB::assertChar('\t', in);
DB::readString(region_name, in);
DB::assertChar('\n', in);
if (read_region_id <= 0)
continue;
RegionID region_id = read_region_id;
size_t old_size = new_chars.size();
if (new_chars.capacity() < old_size + region_name.length() + 1)
throw Poco::Exception("Logical error. Maybe size of file " + path + " is wrong.");
new_chars.resize(old_size + region_name.length() + 1);
memcpy(&new_chars[old_size], region_name.c_str(), region_name.length() + 1);
if (region_id > max_region_id)
{
max_region_id = region_id;
if (region_id > max_size)
throw DB::Exception("Region id is too large: " + DB::toString(region_id) + ", should be not more than " + DB::toString(max_size));
}
while (region_id >= new_names_refs.size())
new_names_refs.resize(new_names_refs.size() * 2, StringRef("", 0));
new_names_refs[region_id] = StringRef(&new_chars[old_size], region_name.length());
}
chars[language_id].swap(new_chars);
names_refs[language_id].swap(new_names_refs);
}
for (size_t language_id = 0; language_id < SUPPORTED_LANGUAGES_COUNT; ++language_id)
names_refs[language_id].resize(max_region_id + 1, StringRef("", 0));
}
StringRef getRegionName(RegionID region_id, Language language = Language::RU) const
{
@ -192,24 +106,10 @@ public:
throw Poco::Exception("Unsupported language for region name. Supported languages are: " + dumpSupportedLanguagesNames() + ".");
}
static std::string dumpSupportedLanguagesNames()
{
std::string res = "";
for (size_t i = 0; i < LANGUAGE_ALIASES_COUNT; ++i)
{
if (i > 0)
res += ", ";
res += '\'';
res += getLanguageAliases()[i].name;
res += '\'';
}
return res;
}
private:
const std::string directory;
static std::string dumpSupportedLanguagesNames();
ModificationTimes file_modification_times = ModificationTimes(SUPPORTED_LANGUAGES_COUNT);
Logger * log = &Logger::get("RegionsNames");
/// Байты имен для каждого языка, уложенные подряд, разделенные нулями
CharsForLanguageID chars = CharsForLanguageID(SUPPORTED_LANGUAGES_COUNT);

67
dbms/include/DB/Dictionaries/Embedded/TechDataHierarchy.h
View File

@ -1,11 +1,7 @@
#pragma once
#include <string.h>
#include <common/logger_useful.h>
#include <common/singleton.h>
#include <mysqlxx/PoolWithFailover.h>
#include <common/Common.h>
/** @brief Класс, позволяющий узнавать, принадлежит ли поисковая система или операционная система
@ -15,57 +11,14 @@
class TechDataHierarchy
{
private:
Logger * log;
UInt8 os_parent[256] {};
UInt8 se_parent[256] {};
UInt8 os_parent[256];
UInt8 se_parent[256];
public:
static constexpr auto required_key = "mysql_metrica";
void reload();
TechDataHierarchy()
: log(&Logger::get("TechDataHierarchy"))
{
LOG_DEBUG(log, "Loading tech data hierarchy.");
memset(os_parent, 0, sizeof(os_parent));
memset(se_parent, 0, sizeof(se_parent));
mysqlxx::PoolWithFailover pool(required_key);
mysqlxx::Pool::Entry conn = pool.Get();
{
mysqlxx::Query q = conn->query("SELECT Id, COALESCE(Parent_Id, 0) FROM OS2");
LOG_TRACE(log, q.str());
mysqlxx::UseQueryResult res = q.use();
while (mysqlxx::Row row = res.fetch())
{
UInt64 child = row[0].getUInt();
UInt64 parent = row[1].getUInt();
if (child > 255 || parent > 255)
throw Poco::Exception("Too large OS id (> 255).");
os_parent[child] = parent;
}
}
{
mysqlxx::Query q = conn->query("SELECT Id, COALESCE(ParentId, 0) FROM SearchEngines");
LOG_TRACE(log, q.str());
mysqlxx::UseQueryResult res = q.use();
while (mysqlxx::Row row = res.fetch())
{
UInt64 child = row[0].getUInt();
UInt64 parent = row[1].getUInt();
if (child > 255 || parent > 255)
throw Poco::Exception("Too large search engine id (> 255).");
se_parent[child] = parent;
}
}
}
/// Has corresponding section in configuration file.
static bool isConfigured();
/// Отношение "принадлежит".
@ -73,7 +26,7 @@ public:
{
while (lhs != rhs && os_parent[lhs])
lhs = os_parent[lhs];
return lhs == rhs;
}
@ -84,8 +37,8 @@ public:
return lhs == rhs;
}
UInt8 OSToParent(UInt8 x) const
{
return os_parent[x];
@ -95,7 +48,7 @@ public:
{
return se_parent[x];
}
/// К самому верхнему предку.
UInt8 OSToMostAncestor(UInt8 x) const

10
dbms/include/DB/Dictionaries/ExecutableDictionarySource.h
View File

@ -3,10 +3,13 @@
#include <DB/Dictionaries/IDictionarySource.h>
#include <DB/Dictionaries/DictionaryStructure.h>
namespace Poco { class Logger; }
namespace DB
{
/// Allows loading dictionaries from executable
class ExecutableDictionarySource final : public IDictionarySource
{
@ -16,8 +19,7 @@ public:
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
Block & sample_block,
const Context & context
);
const Context & context);
ExecutableDictionarySource(const ExecutableDictionarySource & other);
@ -37,7 +39,7 @@ public:
std::string toString() const override;
private:
Logger * log = &Logger::get("ExecutableDictionarySource");
Poco::Logger * log;
const DictionaryStructure dict_struct;
const std::string command;

33
dbms/include/DB/Dictionaries/ExternalQueryBuilder.h
View File

@ -11,7 +11,7 @@ struct DictionaryStructure;
class WriteBuffer;
/** Генерирует запрос для загрузки данных из внешней БД.
/** Builds a query to load data from external database.
*/
struct ExternalQueryBuilder
{
@ -20,21 +20,33 @@ struct ExternalQueryBuilder
const std::string & table;
const std::string & where;
/// Method to quote identifiers.
/// NOTE There could be differences in escaping rules inside quotes. Escaping rules may not match that required by specific external DBMS.
enum QuotingStyle
{
None, /// Write as-is, without quotes.
Backticks, /// `mysql` style
DoubleQuotes /// "postgres" style
};
QuotingStyle quoting_style;
ExternalQueryBuilder(
const DictionaryStructure & dict_struct,
const std::string & db,
const std::string & table,
const std::string & where);
const std::string & where,
QuotingStyle quoting_style);
/** Получить запрос на загрузку всех данных. */
/** Generate a query to load all data. */
std::string composeLoadAllQuery() const;
/** Получить запрос на загрузку данных по множеству простых ключей. */
/** Generate a query to load data by set of UInt64 keys. */
std::string composeLoadIdsQuery(const std::vector<UInt64> & ids);
/** Получить запрос на загрузку данных по множеству сложных ключей.
* Есть два метода их указания в секции WHERE:
/** Generate a query to load data by set of composite keys.
* There are two methods of specification of composite keys in WHERE:
* 1. (x = c11 AND y = c12) OR (x = c21 AND y = c22) ...
* 2. (x, y) IN ((c11, c12), (c21, c22), ...)
*/
@ -51,14 +63,17 @@ struct ExternalQueryBuilder
private:
/// Выражение вида (x = c1 AND y = c2 ...)
/// Expression in form (x = c1 AND y = c2 ...)
void composeKeyCondition(const ConstColumnPlainPtrs & key_columns, const size_t row, WriteBuffer & out) const;
/// Выражение вида (x, y, ...)
/// Expression in form (x, y, ...)
std::string composeKeyTupleDefinition() const;
/// Выражение вида (c1, c2, ...)
/// Expression in form (c1, c2, ...)
void composeKeyTuple(const ConstColumnPlainPtrs & key_columns, const size_t row, WriteBuffer & out) const;
/// Write string with specified quoting style.
void writeQuoted(const std::string & s, WriteBuffer & out) const;
};
}

7
dbms/include/DB/Dictionaries/HTTPDictionarySource.h
View File

@ -3,10 +3,13 @@
#include <DB/Dictionaries/IDictionarySource.h>
#include <DB/Dictionaries/DictionaryStructure.h>
namespace Poco { class Logger; }
namespace DB
{
/// Allows loading dictionaries from executable
class HTTPDictionarySource final : public IDictionarySource
{
@ -35,7 +38,7 @@ public:
std::string toString() const override;
private:
Logger * log = &Logger::get("HTTPDictionarySource");
Poco::Logger * log;
LocalDateTime getLastModification() const;

8
dbms/include/DB/Dictionaries/MySQLDictionarySource.h
View File

@ -1,14 +1,16 @@
#pragma once
#include <DB/Dictionaries/IDictionarySource.h>
#include <DB/Dictionaries/MySQLBlockInputStream.h>
#include <DB/Dictionaries/ExternalQueryBuilder.h>
#include <DB/Dictionaries/DictionaryStructure.h>
#include <ext/range.hpp>
#include <mysqlxx/Pool.h>
#include <mysqlxx/PoolWithFailover.h>
#include <Poco/Util/AbstractConfiguration.h>
namespace Poco { class Logger; }
namespace DB
{
@ -40,7 +42,7 @@ public:
std::string toString() const override;
private:
Logger * log = &Logger::get("MySQLDictionarySource");
Poco::Logger * log;
static std::string quoteForLike(const std::string s);

4
dbms/include/DB/Dictionaries/ODBCDictionarySource.h
View File

@ -16,6 +16,8 @@ namespace Poco
{
class AbstractConfiguration;
}
class Logger;
}
@ -50,7 +52,7 @@ public:
std::string toString() const override;
private:
Logger * log = &Logger::get("ODBCDictionarySource");
Poco::Logger * log;
const DictionaryStructure dict_struct;
const std::string db;

33
dbms/include/DB/Functions/AccurateComparison.h
View File

@ -1,3 +1,4 @@
#pragma once
#include <DB/Core/Types.h>
/** Preceptually-correct number comparisons.
@ -48,27 +49,27 @@ template <typename TInt, typename TUInt>
using bool_if_le_int_vs_uint_t = std::enable_if_t<is_le_int_vs_uint_t<TInt, TUInt>::value, bool>;
template <typename TInt, typename TUInt>
bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
inline bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
{
return (b > static_cast<TUInt>(std::numeric_limits<TInt>::max()) || a < 0) ? false : static_cast<TUInt>(a) > b;
return static_cast<TUInt>(a) > b && a >= 0 && b <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
template <typename TUInt, typename TInt>
bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
inline bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
{
return (a > static_cast<TUInt>(std::numeric_limits<TInt>::max()) || b < 0) ? true : a > static_cast<TUInt>(b);
return a > static_cast<TUInt>(b) || b < 0 || a > static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
template <typename TInt, typename TUInt>
bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
inline bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
{
return (a < 0 || b > static_cast<TUInt>(std::numeric_limits<TInt>::max())) ? false : static_cast<TUInt>(a) == b;
return static_cast<TUInt>(a) == b && a >= 0 && b <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
template <typename TUInt, typename TInt>
bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
inline bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
return (b < 0 || a > static_cast<TUInt>(std::numeric_limits<TInt>::max())) ? false : a == static_cast<TUInt>(b);
return a == static_cast<TUInt>(b) && b >= 0 && a <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
@ -80,25 +81,25 @@ template <typename TInt, typename TUInt>
using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>::value, bool>;
template <typename TInt, typename TUInt>
bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
{
return static_cast<TInt>(a) > static_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
{
return static_cast<TInt>(a) > static_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
{
return static_cast<TInt>(a) == static_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
return static_cast<TInt>(a) == static_cast<TInt>(b);
}
@ -111,25 +112,25 @@ using bool_if_double_can_be_used = std::enable_if_t<
bool>;
template <typename TAInt, typename TAFloat>
bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAInt a, TAFloat b)
inline bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAInt a, TAFloat b)
{
return static_cast<double>(a) > static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAFloat a, TAInt b)
inline bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAFloat a, TAInt b)
{
return static_cast<double>(a) > static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAInt a, TAFloat b)
inline bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAInt a, TAFloat b)
{
return static_cast<double>(a) == static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAFloat a, TAInt b)
inline bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAFloat a, TAInt b)
{
return static_cast<double>(a) == static_cast<double>(b);
}

12
dbms/include/DB/Functions/FunctionFactory.h
View File

@ -1,6 +1,8 @@
#pragma once
#include <DB/Functions/IFunction.h>
#include <string>
#include <memory>
#include <unordered_map>
#include <common/singleton.h>
@ -8,6 +10,8 @@ namespace DB
{
class Context;
class IFunction;
using FunctionPtr = std::shared_ptr<IFunction>;
/** Creates function by name.
@ -20,13 +24,13 @@ class FunctionFactory : public Singleton<FunctionFactory>
private:
typedef FunctionPtr (*Creator)(const Context & context); /// Not std::function, for lower object size and less indirection.
std::unordered_map<String, Creator> functions;
std::unordered_map<std::string, Creator> functions;
public:
FunctionFactory();
FunctionPtr get(const String & name, const Context & context) const; /// Throws an exception if not found.
FunctionPtr tryGet(const String & name, const Context & context) const; /// Returns nullptr if not found.
FunctionPtr get(const std::string & name, const Context & context) const; /// Throws an exception if not found.
FunctionPtr tryGet(const std::string & name, const Context & context) const; /// Returns nullptr if not found.
/// No locking, you must register all functions before usage of get, tryGet.
template <typename F> void registerFunction()

43
dbms/include/DB/Functions/FunctionsArithmetic.h
View File

@ -5,6 +5,7 @@
#include <DB/DataTypes/DataTypeDateTime.h>
#include <DB/Functions/IFunction.h>
#include <DB/Functions/NumberTraits.h>
#include <DB/Functions/AccurateComparison.h>
#include <DB/Core/FieldVisitors.h>
@ -271,9 +272,9 @@ struct BitShiftRightImpl
template<typename A, typename B>
struct LeastImpl
struct LeastBaseImpl
{
using ResultType = typename NumberTraits::ResultOfIf<A, B>::Type;
using ResultType = NumberTraits::ResultOfLeast<A, B>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
@ -284,9 +285,26 @@ struct LeastImpl
};
template<typename A, typename B>
struct GreatestImpl
struct LeastSpecialImpl
{
using ResultType = typename NumberTraits::ResultOfIf<A, B>::Type;
using ResultType = std::make_signed_t<A>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
static_assert(std::is_same<Result, ResultType>::value, "ResultType != Result");
return accurate::lessOp(a, b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
};
template<typename A, typename B>
using LeastImpl = std::conditional_t<!NumberTraits::LeastGreatestSpecialCase<A, B>::value, LeastBaseImpl<A, B>, LeastSpecialImpl<A, B>>;
template<typename A, typename B>
struct GreatestBaseImpl
{
using ResultType = NumberTraits::ResultOfGreatest<A, B>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
@ -295,6 +313,23 @@ struct GreatestImpl
}
};
template<typename A, typename B>
struct GreatestSpecialImpl
{
using ResultType = std::make_unsigned_t<A>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
static_assert(std::is_same<Result, ResultType>::value, "ResultType != Result");
return accurate::greaterOp(a, b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
};
template<typename A, typename B>
using GreatestImpl = std::conditional_t<!NumberTraits::LeastGreatestSpecialCase<A, B>::value, GreatestBaseImpl<A, B>, GreatestSpecialImpl<A, B>>;
template<typename A>
struct NegateImpl
{

20
dbms/include/DB/Functions/FunctionsArray.h
View File

@ -7,26 +7,15 @@
#include <DB/DataTypes/DataTypeDate.h>
#include <DB/DataTypes/DataTypeDateTime.h>
#include <DB/DataTypes/DataTypeString.h>
#include <DB/DataTypes/DataTypeNullable.h>
#include <DB/Columns/ColumnArray.h>
#include <DB/Columns/ColumnString.h>
#include <DB/Columns/ColumnTuple.h>
#include <DB/Columns/ColumnNullable.h>
#include <DB/Functions/IFunction.h>
#include <DB/Functions/Conditional/CondException.h>
#include <DB/Common/HashTable/HashMap.h>
#include <DB/Common/HashTable/ClearableHashMap.h>
#include <DB/Functions/DataTypeTraits.h>
#include <DB/Common/StringUtils.h>
#include <DB/Interpreters/AggregationCommon.h>
#include <DB/Functions/FunctionsConditional.h>
#include <DB/Functions/FunctionsConversion.h>
#include <DB/Functions/Conditional/getArrayType.h>
#include <DB/AggregateFunctions/IAggregateFunction.h>
#include <DB/AggregateFunctions/AggregateFunctionFactory.h>
#include <DB/Parsers/ExpressionListParsers.h>
#include <DB/Parsers/parseQuery.h>
#include <DB/Parsers/ASTExpressionList.h>
#include <DB/Parsers/ASTLiteral.h>
#include <ext/range.hpp>
@ -1432,6 +1421,9 @@ private:
};
class IAggregateFunction;
using AggregateFunctionPtr = std::shared_ptr<IAggregateFunction>;
/** Применяет к массиву агрегатную функцию и возвращает её результат.
* Также может быть применена к нескольким массивам одинаковых размеров, если агрегатная функция принимает несколько аргументов.
*/

763
dbms/include/DB/Functions/FunctionsConversion.h
View File

@ -28,6 +28,7 @@
#include <DB/Interpreters/ExpressionActions.h>
#include <DB/Functions/IFunction.h>
#include <DB/Functions/FunctionsMiscellaneous.h>
#include <DB/Functions/FunctionsDateTime.h>
namespace DB
@ -92,236 +93,33 @@ struct ConvertImpl
/** Conversion of Date to DateTime: adding 00:00:00 time component.
*/
template <typename Name>
struct ConvertImpl<DataTypeDate, DataTypeDateTime, Name>
struct ToDateTimeImpl
{
using FromFieldType = DataTypeDate::FieldType;
using ToFieldType = DataTypeDateTime::FieldType;
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
using FromFieldType = DataTypeDate::FieldType;
const auto & date_lut = DateLUT::instance();
if (const ColumnVector<FromFieldType> * col_from = typeid_cast<const ColumnVector<FromFieldType> *>(block.safeGetByPosition(arguments[0]).column.get()))
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
const typename ColumnVector<FromFieldType>::Container_t & vec_from = col_from->getData();
typename ColumnVector<ToFieldType>::Container_t & vec_to = col_to->getData();
size_t size = vec_from.size();
vec_to.resize(size);
for (size_t i = 0; i < size; ++i)
{
vec_to[i] = date_lut.fromDayNum(DayNum_t(vec_from[i]));
}
}
else if (const ColumnConst<FromFieldType> * col_from
= typeid_cast<const ColumnConst<FromFieldType> *>(block.safeGetByPosition(arguments[0]).column.get()))
{
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToFieldType>>(
col_from->size(), date_lut.fromDayNum(DayNum_t(col_from->getData())));
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
return time_zone.fromDayNum(DayNum_t(d));
}
};
template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime, Name>
: DateTimeTransformImpl<UInt16, UInt32, ToDateTimeImpl, Name> {};
/// Implementation of toDate function.
namespace details
{
template<typename FromType, typename ToType, template <typename, typename> class Transformation>
class Transformer
{
private:
using Op = Transformation<FromType, ToType>;
public:
static void vector_vector(const PaddedPODArray<FromType> & vec_from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToType> & vec_to)
{
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < vec_from.size(); ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
vec_to[i] = Op::execute(vec_from[i], remote_date_lut);
prev_offset = cur_offset;
}
}
static void vector_constant(const PaddedPODArray<FromType> & vec_from, const std::string & data,
PaddedPODArray<ToType> & vec_to)
{
const auto & remote_date_lut = DateLUT::instance(data);
for (size_t i = 0; i < vec_from.size(); ++i)
vec_to[i] = Op::execute(vec_from[i], remote_date_lut);
}
static void vector_constant(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
for (size_t i = 0; i < vec_from.size(); ++i)
vec_to[i] = Op::execute(vec_from[i], local_date_lut);
}
static void constant_vector(const FromType & from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToType> & vec_to)
{
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < offsets.size(); ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
vec_to[i] = Op::execute(from, remote_date_lut);
prev_offset = cur_offset;
}
}
static void constant_constant(const FromType & from, const std::string & data, ToType & to)
{
const auto & remote_date_lut = DateLUT::instance(data);
to = Op::execute(from, remote_date_lut);
}
static void constant_constant(const FromType & from, ToType & to)
{
const auto & local_date_lut = DateLUT::instance();
to = Op::execute(from, local_date_lut);
}
};
template <typename FromType, template <typename, typename> class Transformation, typename Name>
class ToDateConverter
{
private:
using FromFieldType = typename FromType::FieldType;
using ToFieldType = typename DataTypeDate::FieldType;
using Op = Transformer<FromFieldType, ToFieldType, Transformation>;
public:
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
{
const ColumnPtr source_col = block.safeGetByPosition(arguments[0]).column;
const auto * sources = typeid_cast<const ColumnVector<FromFieldType> *>(source_col.get());
const auto * const_source = typeid_cast<const ColumnConst<FromFieldType> *>(source_col.get());
if (arguments.size() == 1)
{
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
const auto & vec_from = sources->getData();
auto & vec_to = col_to->getData();
size_t size = vec_from.size();
vec_to.resize(size);
Op::vector_constant(vec_from, vec_to);
}
else if (const_source)
{
ToFieldType res;
Op::constant_constant(const_source->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToFieldType>>(const_source->size(), res);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else if (arguments.size() == 2)
{
const ColumnPtr time_zone_col = block.safeGetByPosition(arguments[1]).column;
const auto * time_zones = typeid_cast<const ColumnString *>(time_zone_col.get());
const auto * const_time_zone = typeid_cast<const ColumnConstString *>(time_zone_col.get());
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getData();
auto & vec_to = col_to->getData();
vec_to.resize(vec_from.size());
if (time_zones)
Op::vector_vector(vec_from, time_zones->getChars(), time_zones->getOffsets(), vec_to);
else if (const_time_zone)
Op::vector_constant(vec_from, const_time_zone->getData(), vec_to);
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else if (const_source)
{
if (time_zones)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_to = col_to->getData();
vec_to.resize(time_zones->getOffsets().size());
Op::constant_vector(const_source->getData(), time_zones->getChars(), time_zones->getOffsets(), vec_to);
}
else if (const_time_zone)
{
ToFieldType res;
Op::constant_constant(const_source->getData(), const_time_zone->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToFieldType>>(const_source->size(), res);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("FunctionsConversion: Internal error", ErrorCodes::LOGICAL_ERROR);
}
};
template <typename FromType, typename ToType>
struct ToDateTransform
{
static inline ToType execute(const FromType & from, const DateLUTImpl & date_lut)
{
return date_lut.toDayNum(from);
}
};
template <typename FromType, typename ToType>
struct ToDateTransform32Or64
{
static inline ToType execute(const FromType & from, const DateLUTImpl & date_lut)
static inline ToType execute(const FromType & from, const DateLUTImpl & time_zone)
{
return (from < 0xFFFF) ? from : date_lut.toDayNum(from);
return (from < 0xFFFF) ? from : time_zone.toDayNum(from);
}
};
}
/** Conversion of DateTime to Date: throw off time component.
*/
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name>
: details::ToDateConverter<DataTypeDateTime, details::ToDateTransform, Name> {};
: DateTimeTransformImpl<UInt32, UInt16, ToDateImpl, Name> {};
/** Special case of converting (U)Int32 or (U)Int64 (and also, for convenience, Float32, Float64) to Date.
* If number is less than 65536, then it is treated as DayNum, and if greater or equals, then as unix timestamp.
@ -331,17 +129,17 @@ template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name
* (otherwise such usage would be frequent mistake).
*/
template <typename Name> struct ConvertImpl<DataTypeUInt32, DataTypeDate, Name>
: details::ToDateConverter<DataTypeUInt32, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<UInt32, UInt16, ToDateTransform32Or64<UInt32, UInt16>, Name> {};
template <typename Name> struct ConvertImpl<DataTypeUInt64, DataTypeDate, Name>
: details::ToDateConverter<DataTypeUInt64, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<UInt64, UInt16, ToDateTransform32Or64<UInt64, UInt16>, Name> {};
template <typename Name> struct ConvertImpl<DataTypeInt32, DataTypeDate, Name>
: details::ToDateConverter<DataTypeInt32, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<Int32, UInt16, ToDateTransform32Or64<Int32, UInt16>, Name> {};
template <typename Name> struct ConvertImpl<DataTypeInt64, DataTypeDate, Name>
: details::ToDateConverter<DataTypeInt64, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<Int64, UInt16, ToDateTransform32Or64<Int64, UInt16>, Name> {};
template <typename Name> struct ConvertImpl<DataTypeFloat32, DataTypeDate, Name>
: details::ToDateConverter<DataTypeUInt32, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<Float32, UInt16, ToDateTransform32Or64<Float32, UInt16>, Name> {};
template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDate, Name>
: details::ToDateConverter<DataTypeUInt64, details::ToDateTransform32Or64, Name> {};
: DateTimeTransformImpl<Float64, UInt16, ToDateTransform32Or64<Float64, UInt16>, Name> {};
/** Transformation of numbers, dates, datetimes to strings: through formatting.
@ -349,7 +147,7 @@ template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDate, Name>
template <typename DataType>
struct FormatImpl
{
static void execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType & type = DataType{})
static void execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType * type, const DateLUTImpl * time_zone)
{
writeText(x, wb);
}
@ -358,7 +156,7 @@ struct FormatImpl
template <>
struct FormatImpl<DataTypeDate>
{
static void execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate & type = DataTypeDate{})
static void execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate * type, const DateLUTImpl * time_zone)
{
writeDateText(DayNum_t(x), wb);
}
@ -367,18 +165,18 @@ struct FormatImpl<DataTypeDate>
template <>
struct FormatImpl<DataTypeDateTime>
{
static void execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime &type = DataTypeDateTime{})
static void execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime * type, const DateLUTImpl * time_zone)
{
writeDateTimeText(x, wb);
writeDateTimeText(x, wb, *time_zone);
}
};
template <typename FieldType>
struct FormatImpl<DataTypeEnum<FieldType>>
{
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> & type)
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> * type, const DateLUTImpl * time_zone)
{
writeString(type.getNameForValue(x), wb);
writeString(type->getNameForValue(x), wb);
}
};
@ -394,6 +192,10 @@ struct ConvertImpl<DataTypeEnum<FieldType>, typename DataTypeFromFieldType<Field
};
/// For functions toDateTime, toUnixTimestamp and toString from DateTime type, second argument with time zone could be specified.
const DateLUTImpl * extractTimeZoneFromFunctionArguments(Block & block, const ColumnNumbers & arguments);
template <typename FromDataType, typename Name>
struct ConvertImpl<FromDataType, DataTypeString, Name>
{
@ -404,6 +206,12 @@ struct ConvertImpl<FromDataType, DataTypeString, Name>
const auto & col_with_type_and_name = block.safeGetByPosition(arguments[0]);
const auto & type = static_cast<const FromDataType &>(*col_with_type_and_name.type);
const DateLUTImpl * time_zone = nullptr;
/// For argument of DateTime type, second argument with time zone could be specified.
if (std::is_same<FromDataType, DataTypeDateTime>::value)
time_zone = extractTimeZoneFromFunctionArguments(block, arguments);
if (const auto col_from = typeid_cast<const ColumnVector<FromFieldType> *>(col_with_type_and_name.column.get()))
{
auto col_to = std::make_shared<ColumnString>();
@ -420,7 +228,7 @@ struct ConvertImpl<FromDataType, DataTypeString, Name>
for (size_t i = 0; i < size; ++i)
{
FormatImpl<FromDataType>::execute(vec_from[i], write_buffer, type);
FormatImpl<FromDataType>::execute(vec_from[i], write_buffer, &type, time_zone);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
@ -431,7 +239,7 @@ struct ConvertImpl<FromDataType, DataTypeString, Name>
{
std::vector<char> buf;
WriteBufferFromVector<std::vector<char> > write_buffer(buf);
FormatImpl<FromDataType>::execute(col_from->getData(), write_buffer, type);
FormatImpl<FromDataType>::execute(col_from->getData(), write_buffer, &type, time_zone);
block.safeGetByPosition(result).column = std::make_shared<ColumnConstString>(col_from->size(), std::string(&buf[0], write_buffer.count()));
}
else
@ -491,268 +299,24 @@ struct ConvertImplGenericToString
};
namespace details { namespace {
/** Пусть source_timestamp представляет дату и время в исходном часовом поясе соответствующем
* объекту from_date_lut. Эта функция возвращает timestamp представлящий те же дату и время
* в часовом поясе соответствующем объекту to_date_lut.
*/
time_t convertTimestamp(time_t source_timestamp, const DateLUTImpl & from_date_lut, const DateLUTImpl & to_date_lut)
{
if (&from_date_lut == &to_date_lut)
return source_timestamp;
else
{
const auto & values = from_date_lut.getValues(source_timestamp);
return to_date_lut.makeDateTime(values.year, values.month, values.day_of_month,
from_date_lut.toHour(source_timestamp),
from_date_lut.toMinuteInaccurate(source_timestamp),
from_date_lut.toSecondInaccurate(source_timestamp));
}
}
/** Функции для преобразования даты + времени в строку.
*/
struct DateTimeToStringConverter
{
using FromFieldType = typename DataTypeDateTime::FieldType;
static void vector_vector(const PaddedPODArray<FromFieldType> & vec_from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, ColumnString & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
ColumnString::Chars_t & data_to = vec_to.getChars();
ColumnString::Offsets_t & offsets_to = vec_to.getOffsets();
size_t size = vec_from.size();
data_to.resize(size * 2);
offsets_to.resize(size);
WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < size; ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
auto ti = convertTimestamp(vec_from[i], remote_date_lut, local_date_lut);
FormatImpl<DataTypeDateTime>::execute(ti, write_buffer);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
prev_offset = cur_offset;
}
data_to.resize(write_buffer.count());
}
static void vector_constant(const PaddedPODArray<FromFieldType> & vec_from, const std::string & data,
ColumnString & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
ColumnString::Chars_t & data_to = vec_to.getChars();
ColumnString::Offsets_t & offsets_to = vec_to.getOffsets();
size_t size = vec_from.size();
data_to.resize(size * 2);
offsets_to.resize(size);
WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
for (size_t i = 0; i < size; ++i)
{
auto ti = convertTimestamp(vec_from[i], remote_date_lut, local_date_lut);
FormatImpl<DataTypeDateTime>::execute(ti, write_buffer);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
data_to.resize(write_buffer.count());
}
static void vector_constant(const PaddedPODArray<FromFieldType> & vec_from, ColumnString & vec_to)
{
ColumnString::Chars_t & data_to = vec_to.getChars();
ColumnString::Offsets_t & offsets_to = vec_to.getOffsets();
size_t size = vec_from.size();
data_to.resize(size * 2);
offsets_to.resize(size);
WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
for (size_t i = 0; i < size; ++i)
{
FormatImpl<DataTypeDateTime>::execute(vec_from[i], write_buffer);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
data_to.resize(write_buffer.count());
}
static void constant_vector(FromFieldType from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets,
ColumnString & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
ColumnString::Chars_t & data_to = vec_to.getChars();
ColumnString::Offsets_t & offsets_to = vec_to.getOffsets();
size_t size = offsets.size();
data_to.resize(size * 2);
offsets_to.resize(size);
WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < size; ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
auto ti = convertTimestamp(from, remote_date_lut, local_date_lut);
FormatImpl<DataTypeDateTime>::execute(ti, write_buffer);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
prev_offset = cur_offset;
}
data_to.resize(write_buffer.count());
}
static void constant_constant(FromFieldType from, const std::string & data, std::string & to)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
std::vector<char> buf;
WriteBufferFromVector<std::vector<char> > write_buffer(buf);
auto ti = convertTimestamp(from, remote_date_lut, local_date_lut);
FormatImpl<DataTypeDateTime>::execute(ti, write_buffer);
to = std::string(&buf[0], write_buffer.count());
}
static void constant_constant(FromFieldType from, std::string & to)
{
std::vector<char> buf;
WriteBufferFromVector<std::vector<char> > write_buffer(buf);
FormatImpl<DataTypeDateTime>::execute(from, write_buffer);
to = std::string(&buf[0], write_buffer.count());
}
};
}}
template<typename Name>
struct ConvertImpl<DataTypeDateTime, DataTypeString, Name>
{
using Op = details::DateTimeToStringConverter;
using FromFieldType = Op::FromFieldType;
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
{
const ColumnPtr source_col = block.safeGetByPosition(arguments[0]).column;
const auto * sources = typeid_cast<const ColumnVector<FromFieldType> *>(source_col.get());
const auto * const_source = typeid_cast<const ColumnConst<FromFieldType> *>(source_col.get());
if (arguments.size() == 1)
{
if (sources)
{
auto col_to = std::make_shared<ColumnString>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getData();
auto & vec_to = *col_to;
Op::vector_constant(vec_from, vec_to);
}
else if (const_source)
{
std::string res;
Op::constant_constant(const_source->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConstString>(const_source->size(), res);
}
else
{
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
}
else if (arguments.size() == 2)
{
const ColumnPtr time_zone_col = block.safeGetByPosition(arguments[1]).column;
const auto * time_zones = typeid_cast<const ColumnString *>(time_zone_col.get());
const auto * const_time_zone = typeid_cast<const ColumnConstString *>(time_zone_col.get());
if (sources)
{
auto col_to = std::make_shared<ColumnString>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getData();
auto & vec_to = *col_to;
if (time_zones)
Op::vector_vector(vec_from, time_zones->getChars(), time_zones->getOffsets(), vec_to);
else if (const_time_zone)
Op::vector_constant(vec_from, const_time_zone->getData(), vec_to);
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else if (const_source)
{
if (time_zones)
{
auto col_to = std::make_shared<ColumnString>();
block.safeGetByPosition(result).column = col_to;
auto & vec_to = *col_to;
Op::constant_vector(const_source->getData(), time_zones->getChars(), time_zones->getOffsets(), vec_to);
}
else if (const_time_zone)
{
std::string res;
Op::constant_constant(const_source->getData(), const_time_zone->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConstString>(const_source->size(), res);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Internal error.", ErrorCodes::LOGICAL_ERROR);
}
};
/** Conversion of strings to numbers, dates, datetimes: through parsing.
*/
template <typename DataType> void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb) { readText(x,rb); }
template <typename DataType> void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
readText(x, rb);
}
template <> inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb)
template <> inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
DayNum_t tmp(0);
readDateText(tmp, rb);
x = tmp;
}
template <> inline void parseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb)
template <> inline void parseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
time_t tmp = 0;
readDateTimeText(tmp, rb);
readDateTimeText(tmp, rb, *time_zone);
x = tmp;
}
@ -770,6 +334,12 @@ struct ConvertImpl<DataTypeString, ToDataType, Name>
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
{
const DateLUTImpl * time_zone = nullptr;
/// For conversion to DateTime type, second argument with time zone could be specified.
if (std::is_same<ToDataType, DataTypeDateTime>::value)
time_zone = extractTimeZoneFromFunctionArguments(block, arguments);
if (const ColumnString * col_from = typeid_cast<const ColumnString *>(block.safeGetByPosition(arguments[0]).column.get()))
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
@ -788,7 +358,7 @@ struct ConvertImpl<DataTypeString, ToDataType, Name>
{
ReadBufferFromMemory read_buffer(&chars[current_offset], offsets[i] - current_offset - 1);
parseImpl<ToDataType>(vec_to[i], read_buffer);
parseImpl<ToDataType>(vec_to[i], read_buffer, time_zone);
if (!read_buffer.eof()
&& !(std::is_same<ToDataType, DataTypeDate>::value /// Special exception, that allows to parse string with DateTime as Date.
@ -803,7 +373,7 @@ struct ConvertImpl<DataTypeString, ToDataType, Name>
const String & s = col_from->getData();
ReadBufferFromString read_buffer(s);
ToFieldType x = 0;
parseImpl<ToDataType>(x, read_buffer);
parseImpl<ToDataType>(x, read_buffer, time_zone);
if (!read_buffer.eof()
&& !(std::is_same<ToDataType, DataTypeDate>::value /// Special exception, that allows to parse string with DateTime as Date.
@ -814,7 +384,7 @@ struct ConvertImpl<DataTypeString, ToDataType, Name>
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
};
@ -945,226 +515,13 @@ struct ConvertImplGenericFromString
};
namespace details { namespace {
/** Conversion of strings to timestamp. It allows optional second parameter - time zone.
*/
struct StringToTimestampConverter
{
using ToFieldType = typename DataTypeInt32::FieldType;
static void vector_vector(const ColumnString::Chars_t & vec_from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToFieldType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
ReadBufferFromMemory read_buffer(&vec_from[0], vec_from.size());
ColumnString::Offset_t prev_offset = 0;
char zero = 0;
for (size_t i = 0; i < vec_to.size(); ++i)
{
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
auto ti = convertTimestamp(x, local_date_lut, remote_date_lut);
vec_to[i] = ti;
readChar(zero, read_buffer);
if (zero != 0)
throw Exception("Cannot parse from string.", ErrorCodes::CANNOT_PARSE_NUMBER);
prev_offset = cur_offset;
}
}
static void vector_constant(const ColumnString::Chars_t & vec_from, const std::string & data,
PaddedPODArray<ToFieldType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
ReadBufferFromMemory read_buffer(&vec_from[0], vec_from.size());
char zero = 0;
for (size_t i = 0; i < vec_to.size(); ++i)
{
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
auto ti = convertTimestamp(x, local_date_lut, remote_date_lut);
vec_to[i] = ti;
readChar(zero, read_buffer);
if (zero != 0)
throw Exception("Cannot parse from string.", ErrorCodes::CANNOT_PARSE_NUMBER);
}
}
static void vector_constant(const ColumnString::Chars_t & vec_from, PaddedPODArray<ToFieldType> & vec_to)
{
ReadBufferFromMemory read_buffer(&vec_from[0], vec_from.size());
char zero = 0;
for (size_t i = 0; i < vec_to.size(); ++i)
{
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
vec_to[i] = x;
readChar(zero, read_buffer);
if (zero != 0)
throw Exception("Cannot parse from string.", ErrorCodes::CANNOT_PARSE_NUMBER);
}
}
static void constant_vector(const std::string & from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToFieldType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
ReadBufferFromString read_buffer(from);
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < offsets.size(); ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
auto ti = convertTimestamp(x, local_date_lut, remote_date_lut);
vec_to[i] = ti;
prev_offset = cur_offset;
}
}
static void constant_constant(const std::string & from, const std::string & data, ToFieldType & to)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
ReadBufferFromString read_buffer(from);
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
to = convertTimestamp(x, local_date_lut, remote_date_lut);
}
static void constant_constant(const std::string & from, ToFieldType & to)
{
ReadBufferFromString read_buffer(from);
DataTypeDateTime::FieldType x = 0;
parseImpl<DataTypeDateTime>(x, read_buffer);
to = x;
}
};
}}
/// Function toUnixTimestamp has exactly the same implementation as toDateTime of String type.
/// Note that time zone argument could be passed only for toUnixTimestamp function.
struct NameToUnixTimestamp { static constexpr auto name = "toUnixTimestamp"; };
template <>
struct ConvertImpl<DataTypeString, DataTypeInt32, NameToUnixTimestamp>
{
using Op = details::StringToTimestampConverter;
using ToFieldType = Op::ToFieldType;
static void execute(Block & block, const ColumnNumbers & arguments, size_t result)
{
const ColumnPtr source_col = block.safeGetByPosition(arguments[0]).column;
const auto * sources = typeid_cast<const ColumnString *>(source_col.get());
const auto * const_source = typeid_cast<const ColumnConstString *>(source_col.get());
if (arguments.size() == 1)
{
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getChars();
auto & vec_to = col_to->getData();
size_t size = sources->size();
vec_to.resize(size);
Op::vector_constant(vec_from, vec_to);
}
else if (const_source)
{
ToFieldType res;
Op::constant_constant(const_source->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToFieldType>>(const_source->size(), res);
}
else
{
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + NameToUnixTimestamp::name,
ErrorCodes::ILLEGAL_COLUMN);
}
}
else if (arguments.size() == 2)
{
const ColumnPtr time_zone_col = block.safeGetByPosition(arguments[1]).column;
const auto * time_zones = typeid_cast<const ColumnString *>(time_zone_col.get());
const auto * const_time_zone = typeid_cast<const ColumnConstString *>(time_zone_col.get());
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getChars();
auto & vec_to = col_to->getData();
size_t size = sources->size();
vec_to.resize(size);
if (time_zones)
Op::vector_vector(vec_from, time_zones->getChars(), time_zones->getOffsets(), vec_to);
else if (const_time_zone)
Op::vector_constant(vec_from, const_time_zone->getData(), vec_to);
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + NameToUnixTimestamp::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else if (const_source)
{
if (time_zones)
{
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_to = col_to->getData();
vec_to.resize(time_zones->getOffsets().size());
Op::constant_vector(const_source->getData(), time_zones->getChars(), time_zones->getOffsets(), vec_to);
}
else if (const_time_zone)
{
ToFieldType res;
Op::constant_constant(const_source->getData(), const_time_zone->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToFieldType>>(const_source->size(), res);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + NameToUnixTimestamp::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + NameToUnixTimestamp::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Internal error.", ErrorCodes::LOGICAL_ERROR);
}
};
struct ConvertImpl<DataTypeString, DataTypeUInt32, NameToUnixTimestamp>
: ConvertImpl<DataTypeString, DataTypeDateTime, NameToUnixTimestamp> {};
/** If types are identical, just take reference to column.
@ -1190,6 +547,12 @@ struct ConvertImpl<DataTypeFixedString, ToDataType, Name>
{
if (const ColumnFixedString * col_from = typeid_cast<const ColumnFixedString *>(block.safeGetByPosition(arguments[0]).column.get()))
{
const DateLUTImpl * time_zone = nullptr;
/// For conversion to DateTime type, second argument with time zone could be specified.
if (std::is_same<ToDataType, DataTypeDateTime>::value)
time_zone = extractTimeZoneFromFunctionArguments(block, arguments);
auto col_to = std::make_shared<ColumnVector<ToFieldType>>();
block.safeGetByPosition(result).column = col_to;
@ -1203,7 +566,7 @@ struct ConvertImpl<DataTypeFixedString, ToDataType, Name>
{
ReadBufferFromMemory read_buffer(&data_from[i * n], n);
const char * end = read_buffer.buffer().end();
parseImpl<ToDataType>(vec_to[i], read_buffer);
parseImpl<ToDataType>(vec_to[i], read_buffer, time_zone);
if (!read_buffer.eof())
{
@ -1794,7 +1157,7 @@ using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, Posit
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToIntMonotonicity<UInt16>>;
using FunctionToDateTime = FunctionConvert<DataTypeDateTime, NameToDateTime, ToIntMonotonicity<UInt32>>;
using FunctionToString = FunctionConvert<DataTypeString, NameToString, ToStringMonotonicity>;
using FunctionToUnixTimestamp = FunctionConvert<DataTypeInt32, NameToUnixTimestamp, ToIntMonotonicity<UInt32>>;
using FunctionToUnixTimestamp = FunctionConvert<DataTypeUInt32, NameToUnixTimestamp, ToIntMonotonicity<UInt32>>;
template <typename DataType> struct FunctionTo;
template <> struct FunctionTo<DataTypeUInt8> { using Type = FunctionToUInt8; };

412
dbms/include/DB/Functions/FunctionsDateTime.h
View File

@ -64,17 +64,17 @@ namespace DB
/// Это фактор-преобразование будет говорить, что функция монотонна всюду.
struct ZeroTransform
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut) { return 0; }
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut) { return 0; }
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return 0; }
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return 0; }
};
struct ToDateImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDate(t);
return UInt16(time_zone.toDayNum(t));
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return d;
}
@ -84,11 +84,11 @@ struct ToDateImpl
struct ToStartOfDayImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDate(t);
return time_zone.toDate(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return d;
}
@ -98,13 +98,13 @@ struct ToStartOfDayImpl
struct ToMondayImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfWeek(remote_date_lut.toDayNum(t));
return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t));
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfWeek(DayNum_t(d));
return time_zone.toFirstDayNumOfWeek(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -112,13 +112,13 @@ struct ToMondayImpl
struct ToStartOfMonthImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfMonth(remote_date_lut.toDayNum(t));
return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t));
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfMonth(DayNum_t(d));
return time_zone.toFirstDayNumOfMonth(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -126,13 +126,13 @@ struct ToStartOfMonthImpl
struct ToStartOfQuarterImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfQuarter(remote_date_lut.toDayNum(t));
return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t));
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfQuarter(DayNum_t(d));
return time_zone.toFirstDayNumOfQuarter(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -140,13 +140,13 @@ struct ToStartOfQuarterImpl
struct ToStartOfYearImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfYear(remote_date_lut.toDayNum(t));
return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t));
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toFirstDayNumOfYear(DayNum_t(d));
return time_zone.toFirstDayNumOfYear(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -156,23 +156,12 @@ struct ToStartOfYearImpl
struct ToTimeImpl
{
/// При переводе во время, дату будем приравнивать к 1970-01-02.
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
time_t remote_ts = remote_date_lut.toTime(t) + 86400;
if (&remote_date_lut == &local_date_lut)
return remote_ts;
else
{
const auto & values = remote_date_lut.getValues(remote_ts);
return local_date_lut.makeDateTime(values.year, values.month, values.day_of_month,
remote_date_lut.toHour(remote_ts),
remote_date_lut.toMinuteInaccurate(remote_ts),
remote_date_lut.toSecondInaccurate(remote_ts));
}
return time_zone.toTime(t) + 86400;
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -182,23 +171,11 @@ struct ToTimeImpl
struct ToStartOfMinuteImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
if (&remote_date_lut == &local_date_lut)
return local_date_lut.toStartOfMinuteInaccurate(t);
else
{
time_t remote_ts = remote_date_lut.toTime(t) + 86400;
remote_ts = remote_date_lut.toStartOfMinuteInaccurate(remote_ts);
const auto & values = remote_date_lut.getValues(t);
return local_date_lut.makeDateTime(values.year, values.month, values.day_of_month,
remote_date_lut.toHour(remote_ts),
remote_date_lut.toMinuteInaccurate(remote_ts),
remote_date_lut.toSecondInaccurate(remote_ts));
}
return time_zone.toStartOfMinuteInaccurate(t);
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toStartOfMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -208,23 +185,11 @@ struct ToStartOfMinuteImpl
struct ToStartOfFiveMinuteImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
if (&remote_date_lut == &local_date_lut)
return local_date_lut.toStartOfFiveMinuteInaccurate(t);
else
{
time_t remote_ts = remote_date_lut.toTime(t) + 86400;
remote_ts = remote_date_lut.toStartOfFiveMinuteInaccurate(remote_ts);
const auto & values = remote_date_lut.getValues(t);
return local_date_lut.makeDateTime(values.year, values.month, values.day_of_month,
remote_date_lut.toHour(remote_ts),
remote_date_lut.toMinuteInaccurate(remote_ts),
remote_date_lut.toSecondInaccurate(remote_ts));
}
return time_zone.toStartOfFiveMinuteInaccurate(t);
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toStartOfFiveMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -234,23 +199,11 @@ struct ToStartOfFiveMinuteImpl
struct ToStartOfHourImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
if (&remote_date_lut == &local_date_lut)
return local_date_lut.toStartOfHourInaccurate(t);
else
{
time_t remote_ts = remote_date_lut.toTime(t) + 86400;
remote_ts = remote_date_lut.toStartOfHourInaccurate(remote_ts);
const auto & values = remote_date_lut.getValues(t);
return local_date_lut.makeDateTime(values.year, values.month, values.day_of_month,
remote_date_lut.toHour(remote_ts),
remote_date_lut.toMinuteInaccurate(remote_ts),
remote_date_lut.toSecondInaccurate(remote_ts));
}
return time_zone.toStartOfHourInaccurate(t);
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toStartOfHour", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -260,13 +213,13 @@ struct ToStartOfHourImpl
struct ToYearImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toYear(t);
return time_zone.toYear(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toYear(DayNum_t(d));
return time_zone.toYear(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -274,13 +227,13 @@ struct ToYearImpl
struct ToMonthImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toMonth(t);
return time_zone.toMonth(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toMonth(DayNum_t(d));
return time_zone.toMonth(DayNum_t(d));
}
using FactorTransform = ToStartOfYearImpl;
@ -288,13 +241,13 @@ struct ToMonthImpl
struct ToDayOfMonthImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDayOfMonth(t);
return time_zone.toDayOfMonth(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDayOfMonth(DayNum_t(d));
return time_zone.toDayOfMonth(DayNum_t(d));
}
using FactorTransform = ToStartOfMonthImpl;
@ -302,13 +255,13 @@ struct ToDayOfMonthImpl
struct ToDayOfWeekImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDayOfWeek(t);
return time_zone.toDayOfWeek(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDayOfWeek(DayNum_t(d));
return time_zone.toDayOfWeek(DayNum_t(d));
}
using FactorTransform = ToMondayImpl;
@ -316,11 +269,11 @@ struct ToDayOfWeekImpl
struct ToHourImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toHour(t);
return time_zone.toHour(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toHour", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -330,11 +283,11 @@ struct ToHourImpl
struct ToMinuteImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toMinuteInaccurate(t);
return time_zone.toMinuteInaccurate(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -344,11 +297,11 @@ struct ToMinuteImpl
struct ToSecondImpl
{
static inline UInt8 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toSecondInaccurate(t);
return time_zone.toSecondInaccurate(t);
}
static inline UInt8 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toSecond", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -358,13 +311,13 @@ struct ToSecondImpl
struct ToRelativeYearNumImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toYear(t);
return time_zone.toYear(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toYear(DayNum_t(d));
return time_zone.toYear(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -372,13 +325,13 @@ struct ToRelativeYearNumImpl
struct ToRelativeMonthNumImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeMonthNum(t);
return time_zone.toRelativeMonthNum(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeMonthNum(DayNum_t(d));
return time_zone.toRelativeMonthNum(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -386,13 +339,13 @@ struct ToRelativeMonthNumImpl
struct ToRelativeWeekNumImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeWeekNum(t);
return time_zone.toRelativeWeekNum(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeWeekNum(DayNum_t(d));
return time_zone.toRelativeWeekNum(DayNum_t(d));
}
using FactorTransform = ZeroTransform;
@ -400,11 +353,11 @@ struct ToRelativeWeekNumImpl
struct ToRelativeDayNumImpl
{
static inline UInt16 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toDayNum(t);
return time_zone.toDayNum(t);
}
static inline UInt16 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
{
return static_cast<DayNum_t>(d);
}
@ -415,11 +368,11 @@ struct ToRelativeDayNumImpl
struct ToRelativeHourNumImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeHourNum(t);
return time_zone.toRelativeHourNum(t);
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toRelativeHourNum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -429,11 +382,11 @@ struct ToRelativeHourNumImpl
struct ToRelativeMinuteNumImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return remote_date_lut.toRelativeMinuteNum(t);
return time_zone.toRelativeMinuteNum(t);
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toRelativeMinuteNum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -443,11 +396,11 @@ struct ToRelativeMinuteNumImpl
struct ToRelativeSecondNumImpl
{
static inline UInt32 execute(UInt32 t, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone)
{
return t;
}
static inline UInt32 execute(UInt16 d, const DateLUTImpl & remote_date_lut, const DateLUTImpl & local_date_lut)
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
{
throw Exception("Illegal type Date of argument for function toRelativeSecondNum", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
@ -459,68 +412,22 @@ struct ToRelativeSecondNumImpl
template<typename FromType, typename ToType, typename Transform>
struct Transformer
{
static void vector_vector(const PaddedPODArray<FromType> & vec_from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToType> & vec_to)
static void vector(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to, const DateLUTImpl & time_zone)
{
const auto & local_date_lut = DateLUT::instance();
ColumnString::Offset_t prev_offset = 0;
size_t size = vec_from.size();
vec_to.resize(size);
for (size_t i = 0; i < vec_from.size(); ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
vec_to[i] = Transform::execute(vec_from[i], remote_date_lut, local_date_lut);
prev_offset = cur_offset;
}
for (size_t i = 0; i < size; ++i)
vec_to[i] = Transform::execute(vec_from[i], time_zone);
}
static void vector_constant(const PaddedPODArray<FromType> & vec_from, const std::string & data,
PaddedPODArray<ToType> & vec_to)
static void constant(const FromType & from, ToType & to, const DateLUTImpl & time_zone)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
for (size_t i = 0; i < vec_from.size(); ++i)
vec_to[i] = Transform::execute(vec_from[i], remote_date_lut, local_date_lut);
}
static void vector_constant(const PaddedPODArray<FromType> & vec_from, PaddedPODArray<ToType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
for (size_t i = 0; i < vec_from.size(); ++i)
vec_to[i] = Transform::execute(vec_from[i], local_date_lut, local_date_lut);
}
static void constant_vector(const FromType & from, const ColumnString::Chars_t & data,
const ColumnString::Offsets_t & offsets, PaddedPODArray<ToType> & vec_to)
{
const auto & local_date_lut = DateLUT::instance();
ColumnString::Offset_t prev_offset = 0;
for (size_t i = 0; i < offsets.size(); ++i)
{
ColumnString::Offset_t cur_offset = offsets[i];
const std::string time_zone(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1);
const auto & remote_date_lut = DateLUT::instance(time_zone);
vec_to[i] = Transform::execute(from, remote_date_lut, local_date_lut);
prev_offset = cur_offset;
}
}
static void constant_constant(const FromType & from, const std::string & data, ToType & to)
{
const auto & local_date_lut = DateLUT::instance();
const auto & remote_date_lut = DateLUT::instance(data);
to = Transform::execute(from, remote_date_lut, local_date_lut);
}
static void constant_constant(const FromType & from, ToType & to)
{
const auto & local_date_lut = DateLUT::instance();
to = Transform::execute(from, local_date_lut, local_date_lut);
to = Transform::execute(from, time_zone);
}
};
template <typename FromType, typename ToType, typename Transform, typename Name>
struct DateTimeTransformImpl
{
@ -529,93 +436,47 @@ struct DateTimeTransformImpl
using Op = Transformer<FromType, ToType, Transform>;
const ColumnPtr source_col = block.safeGetByPosition(arguments[0]).column;
const auto * sources = typeid_cast<const ColumnVector<FromType> *>(&*source_col);
const auto * const_source = typeid_cast<const ColumnConst<FromType> *>(&*source_col);
const auto * sources = typeid_cast<const ColumnVector<FromType> *>(source_col.get());
const auto * const_source = typeid_cast<const ColumnConst<FromType> *>(source_col.get());
if (arguments.size() == 1)
const ColumnConstString * time_zone_column = nullptr;
if (arguments.size() == 2)
{
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToType>>();
block.safeGetByPosition(result).column = col_to;
time_zone_column = typeid_cast<const ColumnConstString *>(block.safeGetByPosition(arguments[1]).column.get());
auto & vec_from = sources->getData();
auto & vec_to = col_to->getData();
size_t size = vec_from.size();
vec_to.resize(size);
Op::vector_constant(vec_from, vec_to);
}
else if (const_source)
{
ToType res;
Op::constant_constant(const_source->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToType>>(const_source->size(), res);
}
else
{
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
if (!time_zone_column)
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second (time zone) argument of function " + Name::name + ", must be constant string",
ErrorCodes::ILLEGAL_COLUMN);
}
}
else if (arguments.size() == 2)
const DateLUTImpl & time_zone = time_zone_column
? DateLUT::instance(time_zone_column->getData())
: DateLUT::instance();
if (sources)
{
const ColumnPtr time_zone_col = block.safeGetByPosition(arguments[1]).column;
const auto * time_zones = typeid_cast<const ColumnString *>(&*time_zone_col);
const auto * const_time_zone = typeid_cast<const ColumnConstString *>(&*time_zone_col);
if (sources)
{
auto col_to = std::make_shared<ColumnVector<ToType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_from = sources->getData();
auto & vec_to = col_to->getData();
vec_to.resize(vec_from.size());
if (time_zones)
Op::vector_vector(vec_from, time_zones->getChars(), time_zones->getOffsets(), vec_to);
else if (const_time_zone)
Op::vector_constant(vec_from, const_time_zone->getData(), vec_to);
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else if (const_source)
{
if (time_zones)
{
auto col_to = std::make_shared<ColumnVector<ToType>>();
block.safeGetByPosition(result).column = col_to;
auto & vec_to = col_to->getData();
vec_to.resize(time_zones->getOffsets().size());
Op::constant_vector(const_source->getData(), time_zones->getChars(), time_zones->getOffsets(), vec_to);
}
else if (const_time_zone)
{
ToType res;
Op::constant_constant(const_source->getData(), const_time_zone->getData(), res);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToType>>(const_source->size(), res);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[1]).column->getName()
+ " of second argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
else
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
auto col_to = std::make_shared<ColumnVector<ToType>>();
block.safeGetByPosition(result).column = col_to;
Op::vector(sources->getData(), col_to->getData(), time_zone);
}
else if (const_source)
{
ToType res;
Op::constant(const_source->getData(), res, time_zone);
block.safeGetByPosition(result).column = std::make_shared<ColumnConst<ToType>>(const_source->size(), res);
}
else
throw Exception("Internal error.", ErrorCodes::LOGICAL_ERROR);
{
throw Exception("Illegal column " + block.safeGetByPosition(arguments[0]).column->getName()
+ " of first argument of function " + Name::name,
ErrorCodes::ILLEGAL_COLUMN);
}
}
};
template <typename ToDataType, typename Transform, typename Name>
class FunctionDateOrDateTimeToSomething : public IFunction
{
@ -637,25 +498,22 @@ public:
{
if (arguments.size() == 1)
{
if ((typeid_cast<const DataTypeDate *>(&*arguments[0]) == nullptr) &&
(typeid_cast<const DataTypeDateTime *>(&*arguments[0]) == nullptr))
if (!typeid_cast<const DataTypeDate *>(arguments[0].get())
&& !typeid_cast<const DataTypeDateTime *>(arguments[0].get()))
throw Exception{
"Illegal type " + arguments[0]->getName() + " of argument of function " + getName() +
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
};
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
else if (arguments.size() == 2)
{
if ((typeid_cast<const DataTypeDate *>(&*arguments[0]) != nullptr)
|| (typeid_cast<const DataTypeDateTime *>(&*arguments[0]) == nullptr)
|| (typeid_cast<const DataTypeString *>(&*arguments[1]) == nullptr))
if (!typeid_cast<const DataTypeDateTime *>(arguments[0].get())
|| !typeid_cast<const DataTypeString *>(arguments[1].get()))
throw Exception{
"Function " + getName() + " supports 1 or 2 arguments. The 1st argument "
"must be of type Date or DateTime. The 2nd argument (optional) must be "
"a constant string with timezone name. The timezone argument is allowed "
"only when the 1st argument has the type DateTime",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
};
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
else
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
@ -703,14 +561,14 @@ public:
if (typeid_cast<const DataTypeDate *>(&type))
{
return Transform::FactorTransform::execute(UInt16(left.get<UInt64>()), date_lut, date_lut)
== Transform::FactorTransform::execute(UInt16(right.get<UInt64>()), date_lut, date_lut)
return Transform::FactorTransform::execute(UInt16(left.get<UInt64>()), date_lut)
== Transform::FactorTransform::execute(UInt16(right.get<UInt64>()), date_lut)
? is_monotonic : is_not_monotonic;
}
else
{
return Transform::FactorTransform::execute(UInt32(left.get<UInt64>()), date_lut, date_lut)
== Transform::FactorTransform::execute(UInt32(right.get<UInt64>()), date_lut, date_lut)
return Transform::FactorTransform::execute(UInt32(left.get<UInt64>()), date_lut)
== Transform::FactorTransform::execute(UInt32(right.get<UInt64>()), date_lut)
? is_monotonic : is_not_monotonic;
}
}
@ -825,7 +683,7 @@ public:
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!typeid_cast<const DataTypeDateTime *>(&*arguments[0]))
if (!typeid_cast<const DataTypeDateTime *>(arguments[0].get()))
throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be DateTime.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -958,11 +816,11 @@ public:
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!typeid_cast<const DataTypeDateTime *>(&*arguments[0]))
if (!typeid_cast<const DataTypeDateTime *>(arguments[0].get()))
throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be DateTime.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (!typeid_cast<const DataTypeUInt32 *>(&*arguments[1]))
if (!typeid_cast<const DataTypeUInt32 *>(arguments[1].get()))
throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ". Must be UInt32.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

38
dbms/include/DB/Functions/FunctionsDictionaries.h
View File

@ -13,10 +13,14 @@
#include <DB/Columns/ColumnTuple.h>
#include <DB/Interpreters/Context.h>
#include <DB/Interpreters/Dictionaries.h>
#include <DB/Interpreters/EmbeddedDictionaries.h>
#include <DB/Interpreters/ExternalDictionaries.h>
#include <DB/Functions/IFunction.h>
#include <DB/Dictionaries/Embedded/RegionsHierarchy.h>
#include <DB/Dictionaries/Embedded/RegionsHierarchies.h>
#include <DB/Dictionaries/Embedded/RegionsNames.h>
#include <DB/Dictionaries/Embedded/TechDataHierarchy.h>
#include <DB/Dictionaries/FlatDictionary.h>
#include <DB/Dictionaries/HashedDictionary.h>
#include <DB/Dictionaries/CacheDictionary.h>
@ -553,7 +557,7 @@ struct FunctionRegionToCity :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -562,7 +566,7 @@ struct FunctionRegionToArea :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -571,7 +575,7 @@ struct FunctionRegionToDistrict :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -580,7 +584,7 @@ struct FunctionRegionToCountry :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -589,7 +593,7 @@ struct FunctionRegionToContinent :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -598,7 +602,7 @@ struct FunctionRegionToTopContinent :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -607,7 +611,7 @@ struct FunctionRegionToPopulation :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -616,7 +620,7 @@ struct FunctionOSToRoot :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -625,7 +629,7 @@ struct FunctionSEToRoot :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -634,7 +638,7 @@ struct FunctionRegionIn :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -643,7 +647,7 @@ struct FunctionOSIn :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -652,7 +656,7 @@ struct FunctionSEIn :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -661,7 +665,7 @@ struct FunctionRegionHierarchy :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getRegionsHierarchies());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getRegionsHierarchies());
}
};
@ -670,7 +674,7 @@ struct FunctionOSHierarchy :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -679,7 +683,7 @@ struct FunctionSEHierarchy :
{
static FunctionPtr create(const Context & context)
{
return std::make_shared<base_type>(context.getDictionaries().getTechDataHierarchy());
return std::make_shared<base_type>(context.getEmbeddedDictionaries().getTechDataHierarchy());
}
};
@ -691,7 +695,7 @@ public:
static constexpr auto name = "regionToName";
static FunctionPtr create(const Context & context)
{
return std::make_shared<FunctionRegionToName>(context.getDictionaries().getRegionsNames());
return std::make_shared<FunctionRegionToName>(context.getEmbeddedDictionaries().getRegionsNames());
}
private:

4
dbms/include/DB/Functions/FunctionsHigherOrder.h
View File

@ -676,13 +676,13 @@ public:
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!arguments[0].column)
throw Exception("First argument for function " + getName() + " must be an expression.",
throw Exception("Type of first argument for function " + getName() + " must be an expression.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const ColumnExpression * column_expression = typeid_cast<const ColumnExpression *>(arguments[0].column.get());
if (!column_expression)
throw Exception("First argument for function " + getName() + " must be an expression.",
throw Exception("Column of first argument for function " + getName() + " must be an expression.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
/// Типы остальных аргументов уже проверены в getLambdaArgumentTypes.

Some files were not shown because too many files have changed in this diff Show More