diff --git a/README.md b/README.md index abb27240391..9ac4e4cc9d2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time. [Read more...](https://clickhouse.yandex/) + +[ClickHouse Meetup in Berlin on October 5, 2017](https://events.yandex.com/events/meetings/05-10-2017/) diff --git a/contrib/libzlib-ng/CMakeLists.txt b/contrib/libzlib-ng/CMakeLists.txt index cd2eeb97c55..3d5be9eabf2 100644 --- a/contrib/libzlib-ng/CMakeLists.txt +++ b/contrib/libzlib-ng/CMakeLists.txt @@ -334,7 +334,7 @@ else() add_feature_info(SSE2 1 "Support the SSE2 instruction set, using \"${SSE2FLAG}\"") endif() if(WITH_OPTIM) - if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm") + if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm" AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") set(ZLIB_ARCH_SRCS ${ZLIB_ARCH_SRCS} ${ARCHDIR}/x86.c) endif() if(HAVE_SSE42_INTRIN) @@ -489,15 +489,39 @@ if(MINGW OR MSYS) set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) endif(MINGW OR MSYS) +add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_GZFILE_SRCS} ${ZLIB_ARCH_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_GZFILE_SRCS} ${ZLIB_ARCH_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) +set_target_properties(zlib PROPERTIES SOVERSION 1) + +if(NOT CYGWIN) + # This property causes shared libraries on Linux to have the full version + # encoded into their final filename. We disable this on Cygwin because + # it causes cygz-${ZLIB_FULL_VERSION}.dll to be created when cygz.dll + # seems to be the default. + # + # This has no effect with MSVC, on that platform the version info for + # the DLL comes from the resource file win32/zlib1.rc + set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION}) +endif() + if(UNIX) # On unix-like platforms the library is almost always called libz - set_target_properties(zlibstatic PROPERTIES OUTPUT_NAME z) + set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) + if(NOT APPLE) + set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") + endif() +elseif(MSYS) + # Suppress version number from shared library name + set(CMAKE_SHARED_LIBRARY_NAME_WITH_VERSION 0) +elseif(BUILD_SHARED_LIBS AND WIN32) + # Creates zlib1.dll when building shared library version + set_target_properties(zlib PROPERTIES SUFFIX "1.dll") endif() if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL ) - install(TARGETS zlibstatic + install(TARGETS zlib zlibstatic RUNTIME DESTINATION "${INSTALL_BIN_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ) @@ -505,6 +529,35 @@ endif() if(NOT SKIP_INSTALL_HEADERS AND NOT SKIP_INSTALL_ALL ) install(FILES ${ZLIB_PUBLIC_HDRS} DESTINATION "${INSTALL_INC_DIR}") endif() +if(NOT SKIP_INSTALL_FILES AND NOT SKIP_INSTALL_ALL ) + install(FILES zlib.3 DESTINATION "${INSTALL_MAN_DIR}/man3") +endif() if(NOT SKIP_INSTALL_FILES AND NOT SKIP_INSTALL_ALL ) install(FILES ${ZLIB_PC} DESTINATION "${INSTALL_PKGCONFIG_DIR}") endif() + +#============================================================================ +# Example binaries +#============================================================================ + +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set (CMAKE_EXE_LINKER_FLAGS "") +endif () + +add_executable(example test/example.c) +target_link_libraries(example zlib) +add_test(example example) + +add_executable(minigzip test/minigzip.c) +target_link_libraries(minigzip zlib) + +if(HAVE_OFF64_T) + add_executable(example64 test/example.c) + target_link_libraries(example64 zlib) + set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") + add_test(example64 example64) + + add_executable(minigzip64 test/minigzip.c) + target_link_libraries(minigzip64 zlib) + set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") +endif() diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 7486c7f884c..52b6e9f7b31 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,6 +1,6 @@ # This strings autochanged from release_lib.sh: -set(VERSION_DESCRIBE v1.1.54288-testing) -set(VERSION_REVISION 54288) +set(VERSION_DESCRIBE v1.1.54292-testing) +set(VERSION_REVISION 54292) # end of autochange set (VERSION_MAJOR 1) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp new file mode 100644 index 00000000000..c2906e69ca1 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + if (argument_types.size() != 2) + throw Exception("Incorrect number of arguments for aggregate function " + name + ", should be 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(); +} + +} + +void registerAggregateFunctionSumMap(AggregateFunctionFactory & factory) +{ + factory.registerFunction("sumMap", createAggregateFunctionSumMap); +} + +} diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h new file mode 100644 index 00000000000..482f9ce372c --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -0,0 +1,192 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +struct AggregateFunctionSumMapData +{ + std::map merged_maps; +}; + +/** Aggregate function, that takes two arguments: keys and values, and as a result, builds an array of 2 arrays - + * ordered keys and values summed up by corresponding keys. + * + * This function is the most useful when using SummingMergeTree to sum Nested columns, which name ends in "Map". + * + * Example: sumMap(k, v) of: + * k v + * [1,2,3] [10,10,10] + * [3,4,5] [10,10,10] + * [4,5,6] [10,10,10] + * [6,7,8] [10,10,10] + * [7,5,3] [5,15,25] + * [8,9,10] [20,20,20] + * will return: + * ([1,2,3,4,5,6,7,8,9,10],[10,10,45,20,35,20,15,30,20,20]) + */ +class AggregateFunctionSumMap final : public IBinaryAggregateFunction +{ +private: + DataTypePtr keys_type; + DataTypePtr values_type; + +public: + String getName() const override { return "sumMap"; } + + DataTypePtr getReturnType() const override + { + DataTypes types; + types.emplace_back(std::make_shared(keys_type)); + types.emplace_back(std::make_shared(values_type)); + + return std::make_shared(types); + } + + void setArgumentsImpl(const DataTypes & arguments) + { + if (2 != arguments.size()) + throw Exception("Aggregate function " + getName() + "require exactly two arguments of array type.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const auto * array_type = checkAndGetDataType(arguments[0].get()); + if (!array_type) + throw Exception("First argument for function " + getName() + " must be an array.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + keys_type = array_type->getNestedType(); + + array_type = checkAndGetDataType(arguments[1].get()); + if (!array_type) + throw Exception("Second argument for function " + getName() + " must be an array.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + values_type = array_type->getNestedType(); + } + + void setParameters(const Array & params) override + { + if (!params.empty()) + throw Exception("This instantiation of " + getName() + "aggregate function doesn't accept any parameters.", + ErrorCodes::LOGICAL_ERROR); + } + + void addImpl(AggregateDataPtr place, const IColumn & column_keys, const IColumn & column_values, size_t row_num, Arena *) const + { + Field field_keys; + column_keys.get(row_num, field_keys); + const auto & keys = field_keys.get(); + + Field field_values; + column_values.get(row_num, field_values); + const auto & values = field_values.get(); + + auto & merged_maps = this->data(place).merged_maps; + + if (keys.size() != values.size()) + throw Exception("Sizes of keys and values arrays do not match", ErrorCodes::LOGICAL_ERROR); + + size_t size = keys.size(); + + for (size_t i = 0; i < size; ++i) + { + if (merged_maps.find(keys[i]) != merged_maps.end()) + applyVisitor(FieldVisitorSum(values[i]), merged_maps[keys[i]]); + else + merged_maps[keys[i]] = values[i]; + } + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena * arena) const override + { + auto & merged_maps = this->data(place).merged_maps; + const auto & rhs_maps = this->data(rhs).merged_maps; + + for (const auto &rhs_map : rhs_maps) + { + if (merged_maps.find(rhs_map.first) != merged_maps.end()) + applyVisitor(FieldVisitorSum(rhs_map.second), merged_maps[rhs_map.first]); + else + merged_maps[rhs_map.first] = rhs_map.second; + } + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + const auto & merged_maps = this->data(place).merged_maps; + size_t size = merged_maps.size(); + writeVarUInt(size, buf); + + for (const auto &v : merged_maps) + { + keys_type->serializeBinary(v.first, buf); + values_type->serializeBinary(v.second, buf); + } + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + auto & merged_maps = this->data(place).merged_maps; + + size_t size = 0; + readVarUInt(size, buf); + + for (size_t i = 0; i < size; ++i) + { + Field key, value; + keys_type->deserializeBinary(key, buf); + values_type->deserializeBinary(value, buf); + merged_maps[key] = value; + } + } + + void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + { + auto & to_cols = static_cast(to).getColumns(); + + auto & to_keys_arr = static_cast(*to_cols[0]); + auto & to_values_arr = static_cast(*to_cols[1]); + + auto & to_keys_col = to_keys_arr.getData(); + auto & to_keys_offsets = to_keys_arr.getOffsets(); + + auto & to_values_col = to_values_arr.getData(); + auto & to_values_offsets = to_values_arr.getOffsets(); + + const auto & merged_maps = this->data(place).merged_maps; + size_t size = merged_maps.size(); + + to_keys_col.reserve(size); + to_values_col.reserve(size); + for (const auto &v : merged_maps) + { + to_keys_col.insert(v.first); + to_values_col.insert(v.second); + } + + to_keys_offsets.push_back((to_keys_offsets.empty() ? 0 : to_keys_offsets.back()) + size); + to_values_offsets.push_back((to_values_offsets.empty() ? 0 : to_values_offsets.back()) + size); + } + + const char * getHeaderFilePath() const override { return __FILE__; } +}; + +} diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp index 69f07a38754..30d52a7ad5b 100644 --- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -20,6 +20,7 @@ void registerAggregateFunctionsSequenceMatch(AggregateFunctionFactory & factory) void registerAggregateFunctionsMinMaxAny(AggregateFunctionFactory & factory); void registerAggregateFunctionsStatistics(AggregateFunctionFactory & factory); void registerAggregateFunctionSum(AggregateFunctionFactory & factory); +void registerAggregateFunctionSumMap(AggregateFunctionFactory & factory); void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory); void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory & factory); void registerAggregateFunctionTopK(AggregateFunctionFactory & factory); @@ -45,6 +46,7 @@ void registerAggregateFunctions() registerAggregateFunctionsMinMaxAny(factory); registerAggregateFunctionsStatistics(factory); registerAggregateFunctionSum(factory); + registerAggregateFunctionSumMap(factory); registerAggregateFunctionsUniq(factory); registerAggregateFunctionUniqUpTo(factory); registerAggregateFunctionTopK(factory); diff --git a/dbms/src/Common/ProfileEvents.cpp b/dbms/src/Common/ProfileEvents.cpp index 70bbee64b75..1488d6c8c0d 100644 --- a/dbms/src/Common/ProfileEvents.cpp +++ b/dbms/src/Common/ProfileEvents.cpp @@ -124,8 +124,9 @@ M(DictCacheLockReadNs) \ \ M(DistributedSyncInsertionTimeoutExceeded) \ - M(DataAfterMergeDiffersFromReplica) - + M(DataAfterMergeDiffersFromReplica) \ + M(PolygonsAddedToPool) \ + M(PolygonsInPoolAllocatedBytes) \ namespace ProfileEvents { diff --git a/dbms/src/Common/isLocalAddress.h b/dbms/src/Common/isLocalAddress.h index 6e15490eeed..e6d85432ce8 100644 --- a/dbms/src/Common/isLocalAddress.h +++ b/dbms/src/Common/isLocalAddress.h @@ -1,5 +1,6 @@ #pragma once +#include namespace Poco { diff --git a/dbms/src/Core/FieldVisitors.h b/dbms/src/Core/FieldVisitors.h index 996ebdf4326..de53dc04a03 100644 --- a/dbms/src/Core/FieldVisitors.h +++ b/dbms/src/Core/FieldVisitors.h @@ -14,6 +14,7 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_CONVERT_TYPE; + extern const int LOGICAL_ERROR; } @@ -299,4 +300,23 @@ public: bool operator() (const Tuple & l, const Tuple & r) const { return l < r; } }; +/** Implements `+=` operation. + * Returns false if the result is zero. + */ +class FieldVisitorSum : public StaticVisitor +{ +private: + const Field & rhs; +public: + explicit FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} + + bool operator() (UInt64 & x) const { x += get(rhs); return x != 0; } + bool operator() (Int64 & x) const { x += get(rhs); return x != 0; } + bool operator() (Float64 & x) const { x += get(rhs); return x != 0; } + + bool operator() (Null & x) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); } + bool operator() (String & x) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Array & x) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } +}; + } diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp index e82169d7f77..ca9305f7506 100644 --- a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp @@ -248,27 +248,6 @@ void SummingSortedBlockInputStream::merge(ColumnPlainPtrs & merged_columns, std: finished = true; } - -/** Implements `+=` operation. - * Returns false if the result is zero. - */ -class FieldVisitorSum : public StaticVisitor -{ -private: - const Field & rhs; -public: - explicit FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} - - bool operator() (UInt64 & x) const { x += get(rhs); return x != 0; } - bool operator() (Int64 & x) const { x += get(rhs); return x != 0; } - bool operator() (Float64 & x) const { x += get(rhs); return x != 0; } - - bool operator() (Null & x) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); } - bool operator() (String & x) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Array & x) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } -}; - - template bool SummingSortedBlockInputStream::mergeMaps(Row & row, TSortCursor & cursor) { diff --git a/dbms/src/Functions/FunctionsGeo.cpp b/dbms/src/Functions/FunctionsGeo.cpp index 2dae9288d71..4fd87542098 100644 --- a/dbms/src/Functions/FunctionsGeo.cpp +++ b/dbms/src/Functions/FunctionsGeo.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include @@ -9,36 +11,82 @@ #include #include #include +#include +#include +#include +namespace ProfileEvents +{ + extern const Event PolygonsAddedToPool; + extern const Event PolygonsInPoolAllocatedBytes; +} namespace DB { + namespace ErrorCodes { extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace FunctionPointInPolygonDetail +{ -template -using PointInPolygonCrossing = boost::geometry::strategy::within::crossings_multiply; -template -using PointInPolygonWinding = boost::geometry::strategy::within::winding; -template -using PointInPolygonFranklin = boost::geometry::strategy::within::franklin; +template +ColumnPtr callPointInPolygonImplWithPool(const IColumn & x, const IColumn & y, Polygon & polygon) +{ + using Pool = ObjectPoolMap; + /// C++11 has thread-safe function-local statics on most modern compilers. + static Pool known_polygons; -template