Merged with master

.gitmodules

@@ -104,6 +104,10 @@
 [submodule "contrib/sparsehash-c11"]
 	path = contrib/sparsehash-c11
 	url = https://github.com/sparsehash/sparsehash-c11.git
+[submodule "contrib/grpc"]
+	path = contrib/grpc
+	url = https://github.com/ClickHouse-Extras/grpc.git
+	branch = v1.25.0
 [submodule "contrib/aws"]
 	path = contrib/aws
 	url = https://github.com/aws/aws-sdk-cpp.git

CMakeLists.txt

@@ -229,6 +229,10 @@ endif ()
 # Using system libs can cause a lot of warnings in includes (on macro expansion).
 if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
     option (NO_WERROR "Disable -Werror compiler option" ON)
+
+    if (NOT NO_WERROR)
+        add_warning(error)
+    endif ()
 endif ()
 
 # Make this extra-checks for correct library dependencies.
@@ -326,7 +330,6 @@ include (cmake/find/poco.cmake)
 include (cmake/find/lz4.cmake)
 include (cmake/find/xxhash.cmake)
 include (cmake/find/sparsehash.cmake)
-include (cmake/find/rt.cmake)
 include (cmake/find/execinfo.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
@@ -341,6 +344,7 @@ endif()
 include (cmake/find/libxml2.cmake)
 include (cmake/find/brotli.cmake)
 include (cmake/find/protobuf.cmake)
+include (cmake/find/grpc.cmake)
 include (cmake/find/pdqsort.cmake)
 include (cmake/find/hdfs3.cmake) # uses protobuf
 include (cmake/find/s3.cmake)
@@ -364,9 +368,9 @@ if (ENABLE_TESTS)
 endif ()
 
 # Need to process before "contrib" dir:
-include (libs/libcommon/cmake/find_jemalloc.cmake)
-include (libs/libcommon/cmake/find_cctz.cmake)
-include (libs/libmysqlxx/cmake/find_mysqlclient.cmake)
+include (cmake/find/jemalloc.cmake)
+include (cmake/find/cctz.cmake)
+include (cmake/find/mysqlclient.cmake)
 
 # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
 
@@ -398,7 +402,7 @@ macro (add_executable target)
     endif()
 endmacro()
 
-add_subdirectory (libs)
+add_subdirectory (base)
 add_subdirectory (utils)
 add_subdirectory (dbms)
 

base/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_subdirectory (common)
+add_subdirectory (loggers)
+add_subdirectory (daemon)
+
+if (USE_INTERNAL_MEMCPY)
+    add_subdirectory (memcpy)
+endif()
+
+if (USE_MYSQL)
+    add_subdirectory (mysqlxx)
+endif ()
+
+if (USE_INTERNAL_CONSISTENT_HASHING_LIBRARY)
+    add_subdirectory (consistent-hashing)
+endif ()
+add_subdirectory (consistent-hashing-sumbur)
+add_subdirectory (widechar_width)

base/common/CMakeLists.txt

@@ -0,0 +1,64 @@
+configure_file (config_common.h.in config_common.h)
+
+set (SRCS
+    argsToConfig.cpp
+    coverage.cpp
+    DateLUT.cpp
+    DateLUTImpl.cpp
+    demangle.cpp
+    getMemoryAmount.cpp
+    getThreadId.cpp
+    JSON.cpp
+    LineReader.cpp
+    mremap.cpp
+    phdr_cache.cpp
+    preciseExp10.c
+    setTerminalEcho.cpp
+    shift10.cpp
+    sleep.cpp
+)
+
+if (ENABLE_REPLXX)
+    set (SRCS ${SRCS}
+        ReplxxLineReader.cpp
+        ReplxxLineReader.h
+    )
+endif ()
+
+add_library (common ${SRCS})
+
+target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
+
+if (USE_INTERNAL_MEMCPY)
+    target_link_libraries (common PRIVATE memcpy)
+endif ()
+
+if(CCTZ_INCLUDE_DIR)
+    target_include_directories(common BEFORE PRIVATE ${CCTZ_INCLUDE_DIR})
+endif()
+
+if (NOT USE_INTERNAL_BOOST_LIBRARY)
+    target_include_directories (common SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+endif ()
+
+if(NOT USE_INTERNAL_POCO_LIBRARY)
+    target_include_directories (common SYSTEM BEFORE PUBLIC ${Poco_Foundation_INCLUDE_DIR})
+endif()
+
+if(CCTZ_LIBRARY)
+    target_link_libraries(common PRIVATE ${CCTZ_LIBRARY})
+endif()
+
+target_link_libraries(common PUBLIC replxx)
+
+target_link_libraries (common
+    PUBLIC
+        ${Poco_Util_LIBRARY}
+        ${Poco_Foundation_LIBRARY}
+        ${CITYHASH_LIBRARIES}
+        ${Boost_SYSTEM_LIBRARY}
+)
+
+if (ENABLE_TESTS)
+    add_subdirectory (tests)
+endif ()

base/common/DateLUTImpl.h

@@ -391,9 +391,9 @@ public:
 
     /*
       The bits in week_mode has the following meaning:
-       WeekModeFlag::MONDAY_FIRST (0)  If not set	Sunday is first day of week
-                      If set	Monday is first day of week
-       WeekModeFlag::YEAR (1)	  If not set	Week is in range 0-53
+       WeekModeFlag::MONDAY_FIRST (0)  If not set Sunday is first day of week
+                      If set Monday is first day of week
+       WeekModeFlag::YEAR (1) If not set Week is in range 0-53
 
         Week 0 is returned for the the last week of the previous year (for
         a date at start of january) In this case one can get 53 for the
@@ -401,19 +401,19 @@ public:
         relevant for the given year. Note that this flag is only
         releveant if WeekModeFlag::JANUARY is not set.
 
-                  If set	 Week is in range 1-53.
+                  If set Week is in range 1-53.
 
         In this case one may get week 53 for a date in January (when
         the week is that last week of previous year) and week 1 for a
         date in December.
 
-      WeekModeFlag::FIRST_WEEKDAY (2)  If not set	Weeks are numbered according
+      WeekModeFlag::FIRST_WEEKDAY (2) If not set Weeks are numbered according
                         to ISO 8601:1988
-                  If set	The week that contains the first
+                  If set The week that contains the first
                         'first-day-of-week' is week 1.
 
-      WeekModeFlag::NEWYEAR_DAY (3)   If not set  no meaning
-                  If set	The week that contains the January 1 is week 1.
+      WeekModeFlag::NEWYEAR_DAY (3) If not set no meaning
+                  If set The week that contains the January 1 is week 1.
                             Week is in range 1-53.
                             And ignore WeekModeFlag::YEAR, WeekModeFlag::FIRST_WEEKDAY
 

base/common/JSON.cpp

@@ -399,7 +399,7 @@ JSON::Pos JSON::skipElement() const
 
     ElementType type = getType();
 
-    switch(type)
+    switch (type)
     {
         case TYPE_NULL:
             return skipNull();
@@ -587,7 +587,7 @@ std::string JSON::getString() const
                 ++s;
                 checkPos(s);
 
-                switch(*s)
+                switch (*s)
                 {
                     case '"':
                         buf += '"';
@@ -665,7 +665,7 @@ StringRef JSON::getRawString() const
     if (*s != '"')
         throw JSONException(std::string("JSON: expected \", got ") + *s);
     while (++s != ptr_end && *s != '"');
-    if (s != ptr_end )
+    if (s != ptr_end)
         return StringRef(ptr_begin + 1, s - ptr_begin - 1);
     throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON).");
 }

base/common/LineReader.cpp

@@ -3,8 +3,8 @@
 #include <iostream>
 #include <string_view>
 
-#include <port/unistd.h>
 #include <string.h>
+#include <unistd.h>
 
 #ifdef OS_LINUX
 /// We can detect if code is linked with one or another readline variants or open the library dynamically.

base/common/ReplxxLineReader.cpp

@@ -1,8 +1,8 @@
 #include <common/ReplxxLineReader.h>
 
 #include <errno.h>
-#include <port/unistd.h>
 #include <string.h>
+#include <unistd.h>
 
 namespace
 {

base/common/getMemoryAmount.cpp

@@ -33,7 +33,7 @@ uint64_t getMemoryAmountOrZero()
     /* New 64-bit MEMORYSTATUSEX isn't available.  Use old 32.bit */
     MEMORYSTATUS status;
     status.dwLength = sizeof(status);
-    GlobalMemoryStatus( &status );
+    GlobalMemoryStatus(&status);
     return status.dwTotalPhys;
 
 #elif defined(WIN32) || defined(_WIN32)
@@ -41,7 +41,7 @@ uint64_t getMemoryAmountOrZero()
     /* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */
     MEMORYSTATUSEX status;
     status.dwLength = sizeof(status);
-    GlobalMemoryStatusEx( &status );
+    GlobalMemoryStatusEx(&status);
     return status.ullTotalPhys;
 
 #else
@@ -58,24 +58,24 @@ uint64_t getMemoryAmountOrZero()
 #endif
     uint64_t size = 0; /* 64-bit */
     size_t len = sizeof(size);
-    if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
+    if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
         return size;
-    }
+
     return 0; /* Failed? */
 
 #elif defined(_SC_AIX_REALMEM)
     /* AIX. ----------------------------------------------------- */
-    return sysconf( _SC_AIX_REALMEM ) * 1024;
+    return sysconf(_SC_AIX_REALMEM) * 1024;
 
 #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
     /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
-    return (uint64_t)sysconf( _SC_PHYS_PAGES )
-           * (uint64_t)sysconf( _SC_PAGESIZE );
+    return (uint64_t)sysconf(_SC_PHYS_PAGES)
+           * (uint64_t)sysconf(_SC_PAGESIZE);
 
 #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
     /* Legacy. -------------------------------------------------- */
-    return (uint64_t)sysconf( _SC_PHYS_PAGES )
-           * (uint64_t)sysconf( _SC_PAGE_SIZE );
+    return (uint64_t)sysconf(_SC_PHYS_PAGES)
+           * (uint64_t)sysconf(_SC_PAGE_SIZE);
 
 #elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
     /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
@@ -87,10 +87,10 @@ uint64_t getMemoryAmountOrZero()
     mib[1] = HW_PHYSMEM; /* Others. ------------------ */
 #endif
     unsigned int size = 0; /* 32-bit */
-    size_t len = sizeof( size );
-    if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
+    size_t len = sizeof(size);
+    if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
         return size;
-    }
+
     return 0; /* Failed? */
 #endif /* sysctl and sysconf variants */
 

base/common/shift10.cpp

@@ -1,7 +1,8 @@
 #include <common/shift10.h>
+
 #include <common/likely.h>
+
 #include <limits>
-#include <port/ssize_t.h>
 
 
 template <typename T>

base/common/time.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <time.h>
+
+#if defined (OS_DARWIN)
+#    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
+#endif

base/daemon/include/daemon/BaseDaemon.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <sys/types.h>
-#include <port/unistd.h>
+#include <unistd.h>
 #include <iostream>
 #include <memory>
 #include <functional>

base/daemon/src/BaseDaemon.cpp

@@ -382,7 +382,7 @@ enum class InstructionFail
 
 static std::string instructionFailToString(InstructionFail fail)
 {
-    switch(fail)
+    switch (fail)
     {
         case InstructionFail::NONE:
             return "NONE";

base/ext/singleton.h

@@ -2,7 +2,15 @@
 
 #include <memory>
 
-namespace ext {
+namespace ext
+{
+
+/** Thread-unsafe singleton. It works simply like a global variable.
+  * Supports deinitialization.
+  *
+  * In most of the cases, you don't need this class.
+  * Use "Meyers Singleton" instead: static T & instance() { static T x; return x; }
+  */
 
 template <class T>
 class Singleton
@@ -11,14 +19,7 @@ public:
     Singleton()
     {
         if (!instance)
-            instance.reset(new T);
-    }
-
-    template <typename ... Args>
-    Singleton(const Args & ... args)
-    {
-        instance.reset(new T(args...));
-        /// TODO: throw exception on double-creation.
+            instance = std::make_unique<T>();
     }
 
     T * operator->()