diff --git a/.gitmodules b/.gitmodules index 7d975031c54..6075b7e9243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -128,6 +128,9 @@ [submodule "contrib/icu"] path = contrib/icu url = https://github.com/unicode-org/icu.git +[submodule "contrib/flatbuffers"] + path = contrib/flatbuffers + url = https://github.com/google/flatbuffers.git [submodule "contrib/libc-headers"] path = contrib/libc-headers url = https://github.com/ClickHouse-Extras/libc-headers.git diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index d302fd64e1e..6501a05d7bb 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -54,10 +54,12 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) endif() if(${USE_STATIC_LIBRARIES}) + set(FLATBUFFERS_LIBRARY flatbuffers) set(ARROW_LIBRARY arrow_static) set(PARQUET_LIBRARY parquet_static) set(THRIFT_LIBRARY thrift_static) else() + set(FLATBUFFERS_LIBRARY flatbuffers_shared) set(ARROW_LIBRARY arrow_shared) set(PARQUET_LIBRARY parquet_shared) if(USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) @@ -74,7 +76,7 @@ endif() endif() if(USE_PARQUET) - message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY}") + message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY} ; ${FLATBUFFERS_LIBRARY}") else() message(STATUS "Building without Parquet support") endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 415d3a88703..80ccd8cce58 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -159,6 +159,8 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) set (ARROW_PARQUET ON CACHE INTERNAL "") set (ARROW_VERBOSE_THIRDPARTY_BUILD ON CACHE INTERNAL "") set (ARROW_BUILD_SHARED 1 CACHE INTERNAL "") + set (ARROW_BUILD_UTILITIES OFF CACHE INTERNAL "") + set (ARROW_BUILD_INTEGRATION OFF CACHE INTERNAL "") set (ARROW_BOOST_HEADER_ONLY ON CACHE INTERNAL "") set (Boost_FOUND 1 CACHE INTERNAL "") if (MAKE_STATIC_LIBRARIES) diff --git a/contrib/arrow b/contrib/arrow index 87ac6fddaf2..b789226ccb2 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 87ac6fddaf21d0b4ee8b8090533ff293db0da1b4 +Subproject commit b789226ccb2124285792107c758bb3b40b3d082a diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 3ba24f49498..335106cc7ca 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -1,46 +1,48 @@ +include(ExternalProject) + # === thrift set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp) # contrib/thrift/lib/cpp/CMakeLists.txt set(thriftcpp_SOURCES - ${LIBRARY_DIR}/src/thrift/TApplicationException.cpp - ${LIBRARY_DIR}/src/thrift/TOutput.cpp - ${LIBRARY_DIR}/src/thrift/async/TAsyncChannel.cpp - ${LIBRARY_DIR}/src/thrift/async/TAsyncProtocolProcessor.cpp - ${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.h - ${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp - ${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp - ${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp - ${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp - ${LIBRARY_DIR}/src/thrift/protocol/TJSONProtocol.cpp - ${LIBRARY_DIR}/src/thrift/protocol/TMultiplexedProtocol.cpp - ${LIBRARY_DIR}/src/thrift/protocol/TProtocol.cpp - ${LIBRARY_DIR}/src/thrift/transport/TTransportException.cpp - ${LIBRARY_DIR}/src/thrift/transport/TFDTransport.cpp - ${LIBRARY_DIR}/src/thrift/transport/TSimpleFileTransport.cpp - ${LIBRARY_DIR}/src/thrift/transport/THttpTransport.cpp - ${LIBRARY_DIR}/src/thrift/transport/THttpClient.cpp - ${LIBRARY_DIR}/src/thrift/transport/THttpServer.cpp - ${LIBRARY_DIR}/src/thrift/transport/TSocket.cpp - ${LIBRARY_DIR}/src/thrift/transport/TSocketPool.cpp - ${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp - ${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp - ${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp - ${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp - ${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp - ${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp - ${LIBRARY_DIR}/src/thrift/server/TThreadPoolServer.cpp - ${LIBRARY_DIR}/src/thrift/server/TThreadedServer.cpp -) -set( thriftcpp_threads_SOURCES - ${LIBRARY_DIR}/src/thrift/concurrency/ThreadFactory.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/Thread.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/Monitor.cpp - ${LIBRARY_DIR}/src/thrift/concurrency/Mutex.cpp -) + ${LIBRARY_DIR}/src/thrift/TApplicationException.cpp + ${LIBRARY_DIR}/src/thrift/TOutput.cpp + ${LIBRARY_DIR}/src/thrift/async/TAsyncChannel.cpp + ${LIBRARY_DIR}/src/thrift/async/TAsyncProtocolProcessor.cpp + ${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.h + ${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp + ${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp + ${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp + ${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp + ${LIBRARY_DIR}/src/thrift/protocol/TJSONProtocol.cpp + ${LIBRARY_DIR}/src/thrift/protocol/TMultiplexedProtocol.cpp + ${LIBRARY_DIR}/src/thrift/protocol/TProtocol.cpp + ${LIBRARY_DIR}/src/thrift/transport/TTransportException.cpp + ${LIBRARY_DIR}/src/thrift/transport/TFDTransport.cpp + ${LIBRARY_DIR}/src/thrift/transport/TSimpleFileTransport.cpp + ${LIBRARY_DIR}/src/thrift/transport/THttpTransport.cpp + ${LIBRARY_DIR}/src/thrift/transport/THttpClient.cpp + ${LIBRARY_DIR}/src/thrift/transport/THttpServer.cpp + ${LIBRARY_DIR}/src/thrift/transport/TSocket.cpp + ${LIBRARY_DIR}/src/thrift/transport/TSocketPool.cpp + ${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp + ${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp + ${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp + ${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp + ${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp + ${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp + ${LIBRARY_DIR}/src/thrift/server/TThreadPoolServer.cpp + ${LIBRARY_DIR}/src/thrift/server/TThreadedServer.cpp + ) +set(thriftcpp_threads_SOURCES + ${LIBRARY_DIR}/src/thrift/concurrency/ThreadFactory.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/Thread.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/Monitor.cpp + ${LIBRARY_DIR}/src/thrift/concurrency/Mutex.cpp + ) add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS}) @@ -70,22 +72,94 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" "${PROTO_DIR}/orc_proto.proto") -# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. + +# === flatbuffers + +############################################################## +# fbs - Step 1: build flatbuffers lib and flatc compiler +############################################################## +set(FLATBUFFERS_SRC_DIR ${ClickHouse_SOURCE_DIR}/contrib/flatbuffers) +set(FLATBUFFERS_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/flatbuffers) +set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_SRC_DIR}/include) +set(FLATBUFFERS_COMPILER "${FLATBUFFERS_BINARY_DIR}/flatc") + +# set flatbuffers CMake options +if (${USE_STATIC_LIBRARIES}) + set(FLATBUFFERS_BUILD_FLATLIB ON CACHE BOOL "Enable the build of the flatbuffers library") + set(FLATBUFFERS_BUILD_SHAREDLIB OFF CACHE BOOL "Disable the build of the flatbuffers shared library") +else () + set(FLATBUFFERS_BUILD_SHAREDLIB ON CACHE BOOL "Enable the build of the flatbuffers shared library") + set(FLATBUFFERS_BUILD_FLATLIB OFF CACHE BOOL "Disable the build of the flatbuffers library") +endif () +set(FLATBUFFERS_BUILD_FLATC ON CACHE BOOL "Build flatbuffers compiler") +set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests") + +add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") + +################################### +# fbs - Step 2: compile *.fbs files +################################### +set(ARROW_IPC_SRC_DIR ${ARROW_SRC_DIR}/arrow/ipc) +set(ARROW_FORMAT_SRC_DIR ${ARROW_SRC_DIR}/../../format) + +set(ARROW_GENERATED_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/arrow_gen_headers) +set(FLATBUFFERS_COMPILED_OUT_DIR ${ARROW_GENERATED_INCLUDE_DIR}/arrow/ipc) + +set(FBS_OUTPUT_FILES + "${FLATBUFFERS_COMPILED_OUT_DIR}/File_generated.h" + "${FLATBUFFERS_COMPILED_OUT_DIR}/Message_generated.h" + "${FLATBUFFERS_COMPILED_OUT_DIR}/feather_generated.h" + "${FLATBUFFERS_COMPILED_OUT_DIR}/Schema_generated.h" + "${FLATBUFFERS_COMPILED_OUT_DIR}/SparseTensor_generated.h" + "${FLATBUFFERS_COMPILED_OUT_DIR}/Tensor_generated.h") + +set(FBS_SRC + ${ARROW_FORMAT_SRC_DIR}/Message.fbs + ${ARROW_FORMAT_SRC_DIR}/File.fbs + ${ARROW_FORMAT_SRC_DIR}/Schema.fbs + ${ARROW_FORMAT_SRC_DIR}/Tensor.fbs + ${ARROW_FORMAT_SRC_DIR}/SparseTensor.fbs + ${ARROW_IPC_SRC_DIR}/feather.fbs) + +foreach (FIL ${FBS_SRC}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + list(APPEND ABS_FBS_SRC ${ABS_FIL}) +endforeach () + +message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}, FLATBUFFERS_COMPILER: ${FLATBUFFERS_COMPILER}") +message(STATUS "FLATBUFFERS_COMPILED_OUT_DIR: ${FLATBUFFERS_COMPILED_OUT_DIR}") +message(STATUS "flatc: ${FLATBUFFERS_COMPILER} -c -o ${FLATBUFFERS_COMPILED_OUT_DIR}/ ${ABS_FBS_SRC}") + +add_custom_command(OUTPUT ${FBS_OUTPUT_FILES} + COMMAND ${FLATBUFFERS_COMPILER} + -c + -o + ${FLATBUFFERS_COMPILED_OUT_DIR}/ + ${ABS_FBS_SRC} + DEPENDS flatc ${ABS_FBS_SRC} + COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" + VERBATIM) + +add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES}) +add_dependencies(metadata_fbs flatc) + +# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. # Apple Clang compiler failed to compile this code without specifying c++11 standard. # As result these compiler features detected as absent. In result it failed to compile orc itself. # In orc makefile there is code that sets flags, but arrow-cmake ignores these flags. if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - set (CXX11_FLAGS "-std=c++0x") -endif() + set(CXX11_FLAGS "-std=c++0x") +endif () include(${ClickHouse_SOURCE_DIR}/contrib/orc/cmake_modules/CheckSourceCompiles.cmake) include(orc_check.cmake) -configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") +configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh") set(ORC_SRCS ${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc + ${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc ${ORC_SOURCE_SRC_DIR}/Exceptions.cc ${ORC_SOURCE_SRC_DIR}/OrcFile.cc ${ORC_SOURCE_SRC_DIR}/Reader.cc @@ -119,126 +193,160 @@ set(ORC_SRCS # === arrow set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/arrow) + +configure_file("${LIBRARY_DIR}/util/config.h.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/arrow/util/config.h") + # arrow/cpp/src/arrow/CMakeLists.txt set(ARROW_SRCS - ${LIBRARY_DIR}/array.cc + ${LIBRARY_DIR}/array.cc + ${LIBRARY_DIR}/buffer.cc + ${LIBRARY_DIR}/builder.cc + ${LIBRARY_DIR}/compare.cc + ${LIBRARY_DIR}/extension_type.cc + ${LIBRARY_DIR}/memory_pool.cc + ${LIBRARY_DIR}/pretty_print.cc + ${LIBRARY_DIR}/record_batch.cc + ${LIBRARY_DIR}/result.cc + ${LIBRARY_DIR}/scalar.cc + ${LIBRARY_DIR}/sparse_tensor.cc + ${LIBRARY_DIR}/status.cc + ${LIBRARY_DIR}/table_builder.cc + ${LIBRARY_DIR}/table.cc + ${LIBRARY_DIR}/tensor.cc + ${LIBRARY_DIR}/type.cc + ${LIBRARY_DIR}/visitor.cc - ${LIBRARY_DIR}/builder.cc - ${LIBRARY_DIR}/array/builder_adaptive.cc - ${LIBRARY_DIR}/array/builder_base.cc - ${LIBRARY_DIR}/array/builder_binary.cc - ${LIBRARY_DIR}/array/builder_decimal.cc - ${LIBRARY_DIR}/array/builder_dict.cc - ${LIBRARY_DIR}/array/builder_nested.cc - ${LIBRARY_DIR}/array/builder_primitive.cc + ${LIBRARY_DIR}/array/builder_adaptive.cc + ${LIBRARY_DIR}/array/builder_base.cc + ${LIBRARY_DIR}/array/builder_binary.cc + ${LIBRARY_DIR}/array/builder_decimal.cc + ${LIBRARY_DIR}/array/builder_dict.cc + ${LIBRARY_DIR}/array/builder_nested.cc + ${LIBRARY_DIR}/array/builder_primitive.cc + ${LIBRARY_DIR}/array/builder_union.cc + ${LIBRARY_DIR}/array/concatenate.cc + ${LIBRARY_DIR}/array/dict_internal.cc + ${LIBRARY_DIR}/array/diff.cc - ${LIBRARY_DIR}/buffer.cc - ${LIBRARY_DIR}/compare.cc - ${LIBRARY_DIR}/memory_pool.cc - ${LIBRARY_DIR}/pretty_print.cc - ${LIBRARY_DIR}/record_batch.cc - ${LIBRARY_DIR}/status.cc - ${LIBRARY_DIR}/table.cc - ${LIBRARY_DIR}/table_builder.cc - ${LIBRARY_DIR}/tensor.cc - ${LIBRARY_DIR}/sparse_tensor.cc - ${LIBRARY_DIR}/type.cc - ${LIBRARY_DIR}/visitor.cc + ${LIBRARY_DIR}/csv/converter.cc + ${LIBRARY_DIR}/csv/chunker.cc + ${LIBRARY_DIR}/csv/column_builder.cc + ${LIBRARY_DIR}/csv/options.cc + ${LIBRARY_DIR}/csv/parser.cc + ${LIBRARY_DIR}/csv/reader.cc - ${LIBRARY_DIR}/csv/converter.cc - ${LIBRARY_DIR}/csv/chunker.cc - ${LIBRARY_DIR}/csv/column-builder.cc - ${LIBRARY_DIR}/csv/options.cc - ${LIBRARY_DIR}/csv/parser.cc - ${LIBRARY_DIR}/csv/reader.cc + ${LIBRARY_DIR}/ipc/dictionary.cc + ${LIBRARY_DIR}/ipc/feather.cc + ${LIBRARY_DIR}/ipc/message.cc + ${LIBRARY_DIR}/ipc/metadata_internal.cc + ${LIBRARY_DIR}/ipc/options.cc + ${LIBRARY_DIR}/ipc/reader.cc + ${LIBRARY_DIR}/ipc/writer.cc - ${LIBRARY_DIR}/io/buffered.cc - ${LIBRARY_DIR}/io/compressed.cc - ${LIBRARY_DIR}/io/file.cc - ${LIBRARY_DIR}/io/interfaces.cc - ${LIBRARY_DIR}/io/memory.cc - ${LIBRARY_DIR}/io/readahead.cc + ${LIBRARY_DIR}/io/buffered.cc + ${LIBRARY_DIR}/io/compressed.cc + ${LIBRARY_DIR}/io/file.cc + ${LIBRARY_DIR}/io/interfaces.cc + ${LIBRARY_DIR}/io/memory.cc + ${LIBRARY_DIR}/io/readahead.cc + ${LIBRARY_DIR}/io/slow.cc - ${LIBRARY_DIR}/util/bit-util.cc - ${LIBRARY_DIR}/util/compression.cc - ${LIBRARY_DIR}/util/cpu-info.cc - ${LIBRARY_DIR}/util/decimal.cc - ${LIBRARY_DIR}/util/int-util.cc - ${LIBRARY_DIR}/util/io-util.cc - ${LIBRARY_DIR}/util/logging.cc - ${LIBRARY_DIR}/util/key_value_metadata.cc - ${LIBRARY_DIR}/util/task-group.cc - ${LIBRARY_DIR}/util/thread-pool.cc - ${LIBRARY_DIR}/util/trie.cc - ${LIBRARY_DIR}/util/utf8.cc - ${ORC_SRCS} -) + ${LIBRARY_DIR}/util/basic_decimal.cc + ${LIBRARY_DIR}/util/bit_util.cc + ${LIBRARY_DIR}/util/compression.cc + ${LIBRARY_DIR}/util/compression_lz4.cc + ${LIBRARY_DIR}/util/compression_snappy.cc + ${LIBRARY_DIR}/util/compression_zlib.cc + ${LIBRARY_DIR}/util/compression_zstd.cc + ${LIBRARY_DIR}/util/cpu_info.cc + ${LIBRARY_DIR}/util/decimal.cc + ${LIBRARY_DIR}/util/int_util.cc + ${LIBRARY_DIR}/util/io_util.cc + ${LIBRARY_DIR}/util/key_value_metadata.cc + ${LIBRARY_DIR}/util/logging.cc + ${LIBRARY_DIR}/util/memory.cc + ${LIBRARY_DIR}/util/string_builder.cc + ${LIBRARY_DIR}/util/string.cc + ${LIBRARY_DIR}/util/task_group.cc + ${LIBRARY_DIR}/util/thread_pool.cc + ${LIBRARY_DIR}/util/trie.cc + ${LIBRARY_DIR}/util/utf8.cc + + ${LIBRARY_DIR}/vendored/base64.cpp + ${ORC_SRCS} + ) set(ARROW_SRCS ${ARROW_SRCS} - ${LIBRARY_DIR}/compute/context.cc - ${LIBRARY_DIR}/compute/kernels/boolean.cc - ${LIBRARY_DIR}/compute/kernels/cast.cc - ${LIBRARY_DIR}/compute/kernels/hash.cc - ${LIBRARY_DIR}/compute/kernels/util-internal.cc -) + ${LIBRARY_DIR}/compute/context.cc + ${LIBRARY_DIR}/compute/kernels/boolean.cc + ${LIBRARY_DIR}/compute/kernels/cast.cc + ${LIBRARY_DIR}/compute/kernels/hash.cc + ${LIBRARY_DIR}/compute/kernels/util_internal.cc + ) if (LZ4_INCLUDE_DIR AND LZ4_LIBRARY) set(ARROW_WITH_LZ4 1) -endif() +endif () -if(SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) +if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) set(ARROW_WITH_SNAPPY 1) -endif() +endif () -if(ZLIB_INCLUDE_DIR AND ZLIB_LIBRARIES) +if (ZLIB_INCLUDE_DIR AND ZLIB_LIBRARIES) set(ARROW_WITH_ZLIB 1) -endif() +endif () if (ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) set(ARROW_WITH_ZSTD 1) -endif() +endif () if (ARROW_WITH_LZ4) - add_definitions(-DARROW_WITH_LZ4) - SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_lz4.cc ${ARROW_SRCS}) -endif() + add_definitions(-DARROW_WITH_LZ4) + SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_lz4.cc ${ARROW_SRCS}) +endif () if (ARROW_WITH_SNAPPY) - add_definitions(-DARROW_WITH_SNAPPY) - SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_snappy.cc ${ARROW_SRCS}) -endif() + add_definitions(-DARROW_WITH_SNAPPY) + SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_snappy.cc ${ARROW_SRCS}) +endif () if (ARROW_WITH_ZLIB) - add_definitions(-DARROW_WITH_ZLIB) - SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_zlib.cc ${ARROW_SRCS}) -endif() + add_definitions(-DARROW_WITH_ZLIB) + SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_zlib.cc ${ARROW_SRCS}) +endif () if (ARROW_WITH_ZSTD) - add_definitions(-DARROW_WITH_ZSTD) - SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_zstd.cc ${ARROW_SRCS}) -endif() + add_definitions(-DARROW_WITH_ZSTD) + SET(ARROW_SRCS ${LIBRARY_DIR}/util/compression_zstd.cc ${ARROW_SRCS}) +endif () add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) +# Arrow dependencies +add_dependencies(${ARROW_LIBRARY} ${FLATBUFFERS_LIBRARY} metadata_fbs) + +target_link_libraries(${ARROW_LIBRARY} PRIVATE boost_system_internal boost_filesystem_internal boost_regex_internal) +target_link_libraries(${ARROW_LIBRARY} PRIVATE ${FLATBUFFERS_LIBRARY}) + if (USE_INTERNAL_PROTOBUF_LIBRARY) add_dependencies(${ARROW_LIBRARY} protoc) -endif() +endif () target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src ${Boost_INCLUDE_DIRS}) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) if (ARROW_WITH_LZ4) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${LZ4_LIBRARY}) -endif() +endif () if (ARROW_WITH_SNAPPY) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${SNAPPY_LIBRARY}) -endif() +endif () if (ARROW_WITH_ZLIB) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZLIB_LIBRARIES}) -endif() +endif () if (ARROW_WITH_ZSTD) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY}) -endif() +endif () target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_INCLUDE_DIR}) target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_SOURCE_SRC_DIR}) @@ -248,52 +356,56 @@ target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_SRC_DIR}) target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_DIR}) target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR}) target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${FLATBUFFERS_INCLUDE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_GENERATED_INCLUDE_DIR}) # === parquet set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/parquet) # arrow/cpp/src/parquet/CMakeLists.txt set(PARQUET_SRCS - ${LIBRARY_DIR}/arrow/reader.cc - ${LIBRARY_DIR}/arrow/record_reader.cc - ${LIBRARY_DIR}/arrow/schema.cc - ${LIBRARY_DIR}/arrow/writer.cc - ${LIBRARY_DIR}/bloom_filter.cc - ${LIBRARY_DIR}/column_reader.cc - ${LIBRARY_DIR}/column_scanner.cc - ${LIBRARY_DIR}/column_writer.cc - ${LIBRARY_DIR}/file_reader.cc - ${LIBRARY_DIR}/file_writer.cc - ${LIBRARY_DIR}/metadata.cc - ${LIBRARY_DIR}/murmur3.cc - ${LIBRARY_DIR}/printer.cc - ${LIBRARY_DIR}/schema.cc - ${LIBRARY_DIR}/statistics.cc - ${LIBRARY_DIR}/types.cc - ${LIBRARY_DIR}/util/comparison.cc - ${LIBRARY_DIR}/util/memory.cc -) + ${LIBRARY_DIR}/arrow/reader.cc + ${LIBRARY_DIR}/arrow/reader_internal.cc + ${LIBRARY_DIR}/arrow/schema.cc + ${LIBRARY_DIR}/arrow/writer.cc + ${LIBRARY_DIR}/bloom_filter.cc + ${LIBRARY_DIR}/column_reader.cc + ${LIBRARY_DIR}/column_scanner.cc + ${LIBRARY_DIR}/column_writer.cc + ${LIBRARY_DIR}/deprecated_io.cc + ${LIBRARY_DIR}/encoding.cc + ${LIBRARY_DIR}/file_reader.cc + ${LIBRARY_DIR}/file_writer.cc + ${LIBRARY_DIR}/metadata.cc + ${LIBRARY_DIR}/murmur3.cc + ${LIBRARY_DIR}/platform.cc + ${LIBRARY_DIR}/printer.cc + ${LIBRARY_DIR}/properties.cc + ${LIBRARY_DIR}/schema.cc + ${LIBRARY_DIR}/statistics.cc + ${LIBRARY_DIR}/types.cc + ) #list(TRANSFORM PARQUET_SRCS PREPEND ${LIBRARY_DIR}/) # cmake 3.12 list(APPEND PARQUET_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_constants.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_types.cpp -) + ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_constants.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/parquet/parquet_types.cpp + ) add_library(${PARQUET_LIBRARY} ${PARQUET_SRCS}) target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src) include(${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake) # makes config.h target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY}) target_include_directories(${PARQUET_LIBRARY} PRIVATE ${Boost_INCLUDE_DIRS}) -if(SANITIZE STREQUAL "undefined") +if (SANITIZE STREQUAL "undefined") target_compile_options(${PARQUET_LIBRARY} PRIVATE -fno-sanitize=undefined) target_compile_options(${ARROW_LIBRARY} PRIVATE -fno-sanitize=undefined) -endif() +endif () # === tools set(TOOLS_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet) -set(PARQUET_TOOLS parquet-dump-schema parquet-reader parquet-scan) -foreach(TOOL ${PARQUET_TOOLS}) +set(PARQUET_TOOLS parquet_dump_schema parquet_reader parquet_scan) +foreach (TOOL ${PARQUET_TOOLS}) add_executable(${TOOL} ${TOOLS_DIR}/${TOOL}.cc) target_link_libraries(${TOOL} PRIVATE ${PARQUET_LIBRARY}) -endforeach() +endforeach () diff --git a/contrib/arrow-cmake/cpp/src/arrow/util/config.h b/contrib/arrow-cmake/cpp/src/arrow/util/config.h new file mode 100644 index 00000000000..c803ebb5263 --- /dev/null +++ b/contrib/arrow-cmake/cpp/src/arrow/util/config.h @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#define ARROW_VERSION_MAJOR +#define ARROW_VERSION_MINOR +#define ARROW_VERSION_PATCH +#define ARROW_VERSION ((ARROW_VERSION_MAJOR * 1000) + ARROW_VERSION_MINOR) * 1000 + ARROW_VERSION_PATCH + +/* #undef DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY */ +/* #undef GRPCPP_PP_INCLUDE */ diff --git a/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.cpp b/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.cpp index e1775208368..8efba22eb0e 100644 --- a/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.cpp +++ b/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.cpp @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.11.0) + * Autogenerated by Thrift Compiler (0.12.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated diff --git a/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.h b/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.h index b4fbf3ab99e..a84198bfa44 100644 --- a/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.h +++ b/contrib/arrow-cmake/cpp/src/parquet/parquet_constants.h @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.11.0) + * Autogenerated by Thrift Compiler (0.12.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated diff --git a/contrib/arrow-cmake/cpp/src/parquet/parquet_types.cpp b/contrib/arrow-cmake/cpp/src/parquet/parquet_types.cpp index f9780d268f3..9ad74689567 100644 --- a/contrib/arrow-cmake/cpp/src/parquet/parquet_types.cpp +++ b/contrib/arrow-cmake/cpp/src/parquet/parquet_types.cpp @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.11.0) + * Autogenerated by Thrift Compiler (0.12.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -1167,6 +1167,74 @@ void MicroSeconds::printTo(std::ostream& out) const { } +NanoSeconds::~NanoSeconds() throw() { +} + +std::ostream& operator<<(std::ostream& out, const NanoSeconds& obj) +{ + obj.printTo(out); + return out; +} + + +uint32_t NanoSeconds::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + xfer += iprot->skip(ftype); + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t NanoSeconds::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("NanoSeconds"); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(NanoSeconds &a, NanoSeconds &b) { + using ::std::swap; + (void) a; + (void) b; +} + +NanoSeconds::NanoSeconds(const NanoSeconds& other22) { + (void) other22; +} +NanoSeconds& NanoSeconds::operator=(const NanoSeconds& other23) { + (void) other23; + return *this; +} +void NanoSeconds::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "NanoSeconds("; + out << ")"; +} + + TimeUnit::~TimeUnit() throw() { } @@ -1180,6 +1248,11 @@ void TimeUnit::__set_MICROS(const MicroSeconds& val) { this->MICROS = val; __isset.MICROS = true; } + +void TimeUnit::__set_NANOS(const NanoSeconds& val) { + this->NANOS = val; +__isset.NANOS = true; +} std::ostream& operator<<(std::ostream& out, const TimeUnit& obj) { obj.printTo(out); @@ -1224,6 +1297,14 @@ uint32_t TimeUnit::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 3: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->NANOS.read(iprot); + this->__isset.NANOS = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -1251,6 +1332,11 @@ uint32_t TimeUnit::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += this->MICROS.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.NANOS) { + xfer += oprot->writeFieldBegin("NANOS", ::apache::thrift::protocol::T_STRUCT, 3); + xfer += this->NANOS.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -1260,18 +1346,21 @@ void swap(TimeUnit &a, TimeUnit &b) { using ::std::swap; swap(a.MILLIS, b.MILLIS); swap(a.MICROS, b.MICROS); + swap(a.NANOS, b.NANOS); swap(a.__isset, b.__isset); } -TimeUnit::TimeUnit(const TimeUnit& other22) { - MILLIS = other22.MILLIS; - MICROS = other22.MICROS; - __isset = other22.__isset; +TimeUnit::TimeUnit(const TimeUnit& other24) { + MILLIS = other24.MILLIS; + MICROS = other24.MICROS; + NANOS = other24.NANOS; + __isset = other24.__isset; } -TimeUnit& TimeUnit::operator=(const TimeUnit& other23) { - MILLIS = other23.MILLIS; - MICROS = other23.MICROS; - __isset = other23.__isset; +TimeUnit& TimeUnit::operator=(const TimeUnit& other25) { + MILLIS = other25.MILLIS; + MICROS = other25.MICROS; + NANOS = other25.NANOS; + __isset = other25.__isset; return *this; } void TimeUnit::printTo(std::ostream& out) const { @@ -1279,6 +1368,7 @@ void TimeUnit::printTo(std::ostream& out) const { out << "TimeUnit("; out << "MILLIS="; (__isset.MILLIS ? (out << to_string(MILLIS)) : (out << "")); out << ", " << "MICROS="; (__isset.MICROS ? (out << to_string(MICROS)) : (out << "")); + out << ", " << "NANOS="; (__isset.NANOS ? (out << to_string(NANOS)) : (out << "")); out << ")"; } @@ -1380,13 +1470,13 @@ void swap(TimestampType &a, TimestampType &b) { swap(a.unit, b.unit); } -TimestampType::TimestampType(const TimestampType& other24) { - isAdjustedToUTC = other24.isAdjustedToUTC; - unit = other24.unit; +TimestampType::TimestampType(const TimestampType& other26) { + isAdjustedToUTC = other26.isAdjustedToUTC; + unit = other26.unit; } -TimestampType& TimestampType::operator=(const TimestampType& other25) { - isAdjustedToUTC = other25.isAdjustedToUTC; - unit = other25.unit; +TimestampType& TimestampType::operator=(const TimestampType& other27) { + isAdjustedToUTC = other27.isAdjustedToUTC; + unit = other27.unit; return *this; } void TimestampType::printTo(std::ostream& out) const { @@ -1495,13 +1585,13 @@ void swap(TimeType &a, TimeType &b) { swap(a.unit, b.unit); } -TimeType::TimeType(const TimeType& other26) { - isAdjustedToUTC = other26.isAdjustedToUTC; - unit = other26.unit; +TimeType::TimeType(const TimeType& other28) { + isAdjustedToUTC = other28.isAdjustedToUTC; + unit = other28.unit; } -TimeType& TimeType::operator=(const TimeType& other27) { - isAdjustedToUTC = other27.isAdjustedToUTC; - unit = other27.unit; +TimeType& TimeType::operator=(const TimeType& other29) { + isAdjustedToUTC = other29.isAdjustedToUTC; + unit = other29.unit; return *this; } void TimeType::printTo(std::ostream& out) const { @@ -1610,13 +1700,13 @@ void swap(IntType &a, IntType &b) { swap(a.isSigned, b.isSigned); } -IntType::IntType(const IntType& other28) { - bitWidth = other28.bitWidth; - isSigned = other28.isSigned; +IntType::IntType(const IntType& other30) { + bitWidth = other30.bitWidth; + isSigned = other30.isSigned; } -IntType& IntType::operator=(const IntType& other29) { - bitWidth = other29.bitWidth; - isSigned = other29.isSigned; +IntType& IntType::operator=(const IntType& other31) { + bitWidth = other31.bitWidth; + isSigned = other31.isSigned; return *this; } void IntType::printTo(std::ostream& out) const { @@ -1682,11 +1772,11 @@ void swap(JsonType &a, JsonType &b) { (void) b; } -JsonType::JsonType(const JsonType& other30) { - (void) other30; +JsonType::JsonType(const JsonType& other32) { + (void) other32; } -JsonType& JsonType::operator=(const JsonType& other31) { - (void) other31; +JsonType& JsonType::operator=(const JsonType& other33) { + (void) other33; return *this; } void JsonType::printTo(std::ostream& out) const { @@ -1750,11 +1840,11 @@ void swap(BsonType &a, BsonType &b) { (void) b; } -BsonType::BsonType(const BsonType& other32) { - (void) other32; +BsonType::BsonType(const BsonType& other34) { + (void) other34; } -BsonType& BsonType::operator=(const BsonType& other33) { - (void) other33; +BsonType& BsonType::operator=(const BsonType& other35) { + (void) other35; return *this; } void BsonType::printTo(std::ostream& out) const { @@ -1827,6 +1917,11 @@ void LogicalType::__set_BSON(const BsonType& val) { this->BSON = val; __isset.BSON = true; } + +void LogicalType::__set_UUID(const UUIDType& val) { + this->UUID = val; +__isset.UUID = true; +} std::ostream& operator<<(std::ostream& out, const LogicalType& obj) { obj.printTo(out); @@ -1951,6 +2046,14 @@ uint32_t LogicalType::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 14: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->UUID.read(iprot); + this->__isset.UUID = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -2028,6 +2131,11 @@ uint32_t LogicalType::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += this->BSON.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.UUID) { + xfer += oprot->writeFieldBegin("UUID", ::apache::thrift::protocol::T_STRUCT, 14); + xfer += this->UUID.write(oprot); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -2047,38 +2155,41 @@ void swap(LogicalType &a, LogicalType &b) { swap(a.UNKNOWN, b.UNKNOWN); swap(a.JSON, b.JSON); swap(a.BSON, b.BSON); + swap(a.UUID, b.UUID); swap(a.__isset, b.__isset); } -LogicalType::LogicalType(const LogicalType& other34) { - STRING = other34.STRING; - MAP = other34.MAP; - LIST = other34.LIST; - ENUM = other34.ENUM; - DECIMAL = other34.DECIMAL; - DATE = other34.DATE; - TIME = other34.TIME; - TIMESTAMP = other34.TIMESTAMP; - INTEGER = other34.INTEGER; - UNKNOWN = other34.UNKNOWN; - JSON = other34.JSON; - BSON = other34.BSON; - __isset = other34.__isset; +LogicalType::LogicalType(const LogicalType& other36) { + STRING = other36.STRING; + MAP = other36.MAP; + LIST = other36.LIST; + ENUM = other36.ENUM; + DECIMAL = other36.DECIMAL; + DATE = other36.DATE; + TIME = other36.TIME; + TIMESTAMP = other36.TIMESTAMP; + INTEGER = other36.INTEGER; + UNKNOWN = other36.UNKNOWN; + JSON = other36.JSON; + BSON = other36.BSON; + UUID = other36.UUID; + __isset = other36.__isset; } -LogicalType& LogicalType::operator=(const LogicalType& other35) { - STRING = other35.STRING; - MAP = other35.MAP; - LIST = other35.LIST; - ENUM = other35.ENUM; - DECIMAL = other35.DECIMAL; - DATE = other35.DATE; - TIME = other35.TIME; - TIMESTAMP = other35.TIMESTAMP; - INTEGER = other35.INTEGER; - UNKNOWN = other35.UNKNOWN; - JSON = other35.JSON; - BSON = other35.BSON; - __isset = other35.__isset; +LogicalType& LogicalType::operator=(const LogicalType& other37) { + STRING = other37.STRING; + MAP = other37.MAP; + LIST = other37.LIST; + ENUM = other37.ENUM; + DECIMAL = other37.DECIMAL; + DATE = other37.DATE; + TIME = other37.TIME; + TIMESTAMP = other37.TIMESTAMP; + INTEGER = other37.INTEGER; + UNKNOWN = other37.UNKNOWN; + JSON = other37.JSON; + BSON = other37.BSON; + UUID = other37.UUID; + __isset = other37.__isset; return *this; } void LogicalType::printTo(std::ostream& out) const { @@ -2096,6 +2207,7 @@ void LogicalType::printTo(std::ostream& out) const { out << ", " << "UNKNOWN="; (__isset.UNKNOWN ? (out << to_string(UNKNOWN)) : (out << "")); out << ", " << "JSON="; (__isset.JSON ? (out << to_string(JSON)) : (out << "")); out << ", " << "BSON="; (__isset.BSON ? (out << to_string(BSON)) : (out << "")); + out << ", " << "UUID="; (__isset.UUID ? (out << to_string(UUID)) : (out << "")); out << ")"; } @@ -2183,9 +2295,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast36; - xfer += iprot->readI32(ecast36); - this->type = (Type::type)ecast36; + int32_t ecast38; + xfer += iprot->readI32(ecast38); + this->type = (Type::type)ecast38; this->__isset.type = true; } else { xfer += iprot->skip(ftype); @@ -2201,9 +2313,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast37; - xfer += iprot->readI32(ecast37); - this->repetition_type = (FieldRepetitionType::type)ecast37; + int32_t ecast39; + xfer += iprot->readI32(ecast39); + this->repetition_type = (FieldRepetitionType::type)ecast39; this->__isset.repetition_type = true; } else { xfer += iprot->skip(ftype); @@ -2227,9 +2339,9 @@ uint32_t SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 6: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast38; - xfer += iprot->readI32(ecast38); - this->converted_type = (ConvertedType::type)ecast38; + int32_t ecast40; + xfer += iprot->readI32(ecast40); + this->converted_type = (ConvertedType::type)ecast40; this->__isset.converted_type = true; } else { xfer += iprot->skip(ftype); @@ -2355,31 +2467,31 @@ void swap(SchemaElement &a, SchemaElement &b) { swap(a.__isset, b.__isset); } -SchemaElement::SchemaElement(const SchemaElement& other39) { - type = other39.type; - type_length = other39.type_length; - repetition_type = other39.repetition_type; - name = other39.name; - num_children = other39.num_children; - converted_type = other39.converted_type; - scale = other39.scale; - precision = other39.precision; - field_id = other39.field_id; - logicalType = other39.logicalType; - __isset = other39.__isset; +SchemaElement::SchemaElement(const SchemaElement& other41) { + type = other41.type; + type_length = other41.type_length; + repetition_type = other41.repetition_type; + name = other41.name; + num_children = other41.num_children; + converted_type = other41.converted_type; + scale = other41.scale; + precision = other41.precision; + field_id = other41.field_id; + logicalType = other41.logicalType; + __isset = other41.__isset; } -SchemaElement& SchemaElement::operator=(const SchemaElement& other40) { - type = other40.type; - type_length = other40.type_length; - repetition_type = other40.repetition_type; - name = other40.name; - num_children = other40.num_children; - converted_type = other40.converted_type; - scale = other40.scale; - precision = other40.precision; - field_id = other40.field_id; - logicalType = other40.logicalType; - __isset = other40.__isset; +SchemaElement& SchemaElement::operator=(const SchemaElement& other42) { + type = other42.type; + type_length = other42.type_length; + repetition_type = other42.repetition_type; + name = other42.name; + num_children = other42.num_children; + converted_type = other42.converted_type; + scale = other42.scale; + precision = other42.precision; + field_id = other42.field_id; + logicalType = other42.logicalType; + __isset = other42.__isset; return *this; } void SchemaElement::printTo(std::ostream& out) const { @@ -2465,9 +2577,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast41; - xfer += iprot->readI32(ecast41); - this->encoding = (Encoding::type)ecast41; + int32_t ecast43; + xfer += iprot->readI32(ecast43); + this->encoding = (Encoding::type)ecast43; isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2475,9 +2587,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 3: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast42; - xfer += iprot->readI32(ecast42); - this->definition_level_encoding = (Encoding::type)ecast42; + int32_t ecast44; + xfer += iprot->readI32(ecast44); + this->definition_level_encoding = (Encoding::type)ecast44; isset_definition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2485,9 +2597,9 @@ uint32_t DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast43; - xfer += iprot->readI32(ecast43); - this->repetition_level_encoding = (Encoding::type)ecast43; + int32_t ecast45; + xfer += iprot->readI32(ecast45); + this->repetition_level_encoding = (Encoding::type)ecast45; isset_repetition_level_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2562,21 +2674,21 @@ void swap(DataPageHeader &a, DataPageHeader &b) { swap(a.__isset, b.__isset); } -DataPageHeader::DataPageHeader(const DataPageHeader& other44) { - num_values = other44.num_values; - encoding = other44.encoding; - definition_level_encoding = other44.definition_level_encoding; - repetition_level_encoding = other44.repetition_level_encoding; - statistics = other44.statistics; - __isset = other44.__isset; +DataPageHeader::DataPageHeader(const DataPageHeader& other46) { + num_values = other46.num_values; + encoding = other46.encoding; + definition_level_encoding = other46.definition_level_encoding; + repetition_level_encoding = other46.repetition_level_encoding; + statistics = other46.statistics; + __isset = other46.__isset; } -DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other45) { - num_values = other45.num_values; - encoding = other45.encoding; - definition_level_encoding = other45.definition_level_encoding; - repetition_level_encoding = other45.repetition_level_encoding; - statistics = other45.statistics; - __isset = other45.__isset; +DataPageHeader& DataPageHeader::operator=(const DataPageHeader& other47) { + num_values = other47.num_values; + encoding = other47.encoding; + definition_level_encoding = other47.definition_level_encoding; + repetition_level_encoding = other47.repetition_level_encoding; + statistics = other47.statistics; + __isset = other47.__isset; return *this; } void DataPageHeader::printTo(std::ostream& out) const { @@ -2645,11 +2757,11 @@ void swap(IndexPageHeader &a, IndexPageHeader &b) { (void) b; } -IndexPageHeader::IndexPageHeader(const IndexPageHeader& other46) { - (void) other46; +IndexPageHeader::IndexPageHeader(const IndexPageHeader& other48) { + (void) other48; } -IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other47) { - (void) other47; +IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other49) { + (void) other49; return *this; } void IndexPageHeader::printTo(std::ostream& out) const { @@ -2715,9 +2827,9 @@ uint32_t DictionaryPageHeader::read(::apache::thrift::protocol::TProtocol* iprot break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast48; - xfer += iprot->readI32(ecast48); - this->encoding = (Encoding::type)ecast48; + int32_t ecast50; + xfer += iprot->readI32(ecast50); + this->encoding = (Encoding::type)ecast50; isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -2778,17 +2890,17 @@ void swap(DictionaryPageHeader &a, DictionaryPageHeader &b) { swap(a.__isset, b.__isset); } -DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other49) { - num_values = other49.num_values; - encoding = other49.encoding; - is_sorted = other49.is_sorted; - __isset = other49.__isset; +DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader& other51) { + num_values = other51.num_values; + encoding = other51.encoding; + is_sorted = other51.is_sorted; + __isset = other51.__isset; } -DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other50) { - num_values = other50.num_values; - encoding = other50.encoding; - is_sorted = other50.is_sorted; - __isset = other50.__isset; +DictionaryPageHeader& DictionaryPageHeader::operator=(const DictionaryPageHeader& other52) { + num_values = other52.num_values; + encoding = other52.encoding; + is_sorted = other52.is_sorted; + __isset = other52.__isset; return *this; } void DictionaryPageHeader::printTo(std::ostream& out) const { @@ -2898,9 +3010,9 @@ uint32_t DataPageHeaderV2::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast51; - xfer += iprot->readI32(ecast51); - this->encoding = (Encoding::type)ecast51; + int32_t ecast53; + xfer += iprot->readI32(ecast53); + this->encoding = (Encoding::type)ecast53; isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3019,27 +3131,27 @@ void swap(DataPageHeaderV2 &a, DataPageHeaderV2 &b) { swap(a.__isset, b.__isset); } -DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other52) { - num_values = other52.num_values; - num_nulls = other52.num_nulls; - num_rows = other52.num_rows; - encoding = other52.encoding; - definition_levels_byte_length = other52.definition_levels_byte_length; - repetition_levels_byte_length = other52.repetition_levels_byte_length; - is_compressed = other52.is_compressed; - statistics = other52.statistics; - __isset = other52.__isset; +DataPageHeaderV2::DataPageHeaderV2(const DataPageHeaderV2& other54) { + num_values = other54.num_values; + num_nulls = other54.num_nulls; + num_rows = other54.num_rows; + encoding = other54.encoding; + definition_levels_byte_length = other54.definition_levels_byte_length; + repetition_levels_byte_length = other54.repetition_levels_byte_length; + is_compressed = other54.is_compressed; + statistics = other54.statistics; + __isset = other54.__isset; } -DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other53) { - num_values = other53.num_values; - num_nulls = other53.num_nulls; - num_rows = other53.num_rows; - encoding = other53.encoding; - definition_levels_byte_length = other53.definition_levels_byte_length; - repetition_levels_byte_length = other53.repetition_levels_byte_length; - is_compressed = other53.is_compressed; - statistics = other53.statistics; - __isset = other53.__isset; +DataPageHeaderV2& DataPageHeaderV2::operator=(const DataPageHeaderV2& other55) { + num_values = other55.num_values; + num_nulls = other55.num_nulls; + num_rows = other55.num_rows; + encoding = other55.encoding; + definition_levels_byte_length = other55.definition_levels_byte_length; + repetition_levels_byte_length = other55.repetition_levels_byte_length; + is_compressed = other55.is_compressed; + statistics = other55.statistics; + __isset = other55.__isset; return *this; } void DataPageHeaderV2::printTo(std::ostream& out) const { @@ -3130,9 +3242,9 @@ uint32_t PageHeader::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast54; - xfer += iprot->readI32(ecast54); - this->type = (PageType::type)ecast54; + int32_t ecast56; + xfer += iprot->readI32(ecast56); + this->type = (PageType::type)ecast56; isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3272,27 +3384,27 @@ void swap(PageHeader &a, PageHeader &b) { swap(a.__isset, b.__isset); } -PageHeader::PageHeader(const PageHeader& other55) { - type = other55.type; - uncompressed_page_size = other55.uncompressed_page_size; - compressed_page_size = other55.compressed_page_size; - crc = other55.crc; - data_page_header = other55.data_page_header; - index_page_header = other55.index_page_header; - dictionary_page_header = other55.dictionary_page_header; - data_page_header_v2 = other55.data_page_header_v2; - __isset = other55.__isset; +PageHeader::PageHeader(const PageHeader& other57) { + type = other57.type; + uncompressed_page_size = other57.uncompressed_page_size; + compressed_page_size = other57.compressed_page_size; + crc = other57.crc; + data_page_header = other57.data_page_header; + index_page_header = other57.index_page_header; + dictionary_page_header = other57.dictionary_page_header; + data_page_header_v2 = other57.data_page_header_v2; + __isset = other57.__isset; } -PageHeader& PageHeader::operator=(const PageHeader& other56) { - type = other56.type; - uncompressed_page_size = other56.uncompressed_page_size; - compressed_page_size = other56.compressed_page_size; - crc = other56.crc; - data_page_header = other56.data_page_header; - index_page_header = other56.index_page_header; - dictionary_page_header = other56.dictionary_page_header; - data_page_header_v2 = other56.data_page_header_v2; - __isset = other56.__isset; +PageHeader& PageHeader::operator=(const PageHeader& other58) { + type = other58.type; + uncompressed_page_size = other58.uncompressed_page_size; + compressed_page_size = other58.compressed_page_size; + crc = other58.crc; + data_page_header = other58.data_page_header; + index_page_header = other58.index_page_header; + dictionary_page_header = other58.dictionary_page_header; + data_page_header_v2 = other58.data_page_header_v2; + __isset = other58.__isset; return *this; } void PageHeader::printTo(std::ostream& out) const { @@ -3407,15 +3519,15 @@ void swap(KeyValue &a, KeyValue &b) { swap(a.__isset, b.__isset); } -KeyValue::KeyValue(const KeyValue& other57) { - key = other57.key; - value = other57.value; - __isset = other57.__isset; +KeyValue::KeyValue(const KeyValue& other59) { + key = other59.key; + value = other59.value; + __isset = other59.__isset; } -KeyValue& KeyValue::operator=(const KeyValue& other58) { - key = other58.key; - value = other58.value; - __isset = other58.__isset; +KeyValue& KeyValue::operator=(const KeyValue& other60) { + key = other60.key; + value = other60.value; + __isset = other60.__isset; return *this; } void KeyValue::printTo(std::ostream& out) const { @@ -3544,15 +3656,15 @@ void swap(SortingColumn &a, SortingColumn &b) { swap(a.nulls_first, b.nulls_first); } -SortingColumn::SortingColumn(const SortingColumn& other59) { - column_idx = other59.column_idx; - descending = other59.descending; - nulls_first = other59.nulls_first; +SortingColumn::SortingColumn(const SortingColumn& other61) { + column_idx = other61.column_idx; + descending = other61.descending; + nulls_first = other61.nulls_first; } -SortingColumn& SortingColumn::operator=(const SortingColumn& other60) { - column_idx = other60.column_idx; - descending = other60.descending; - nulls_first = other60.nulls_first; +SortingColumn& SortingColumn::operator=(const SortingColumn& other62) { + column_idx = other62.column_idx; + descending = other62.descending; + nulls_first = other62.nulls_first; return *this; } void SortingColumn::printTo(std::ostream& out) const { @@ -3613,9 +3725,9 @@ uint32_t PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast61; - xfer += iprot->readI32(ecast61); - this->page_type = (PageType::type)ecast61; + int32_t ecast63; + xfer += iprot->readI32(ecast63); + this->page_type = (PageType::type)ecast63; isset_page_type = true; } else { xfer += iprot->skip(ftype); @@ -3623,9 +3735,9 @@ uint32_t PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast62; - xfer += iprot->readI32(ecast62); - this->encoding = (Encoding::type)ecast62; + int32_t ecast64; + xfer += iprot->readI32(ecast64); + this->encoding = (Encoding::type)ecast64; isset_encoding = true; } else { xfer += iprot->skip(ftype); @@ -3686,15 +3798,15 @@ void swap(PageEncodingStats &a, PageEncodingStats &b) { swap(a.count, b.count); } -PageEncodingStats::PageEncodingStats(const PageEncodingStats& other63) { - page_type = other63.page_type; - encoding = other63.encoding; - count = other63.count; +PageEncodingStats::PageEncodingStats(const PageEncodingStats& other65) { + page_type = other65.page_type; + encoding = other65.encoding; + count = other65.count; } -PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other64) { - page_type = other64.page_type; - encoding = other64.encoding; - count = other64.count; +PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats& other66) { + page_type = other66.page_type; + encoding = other66.encoding; + count = other66.count; return *this; } void PageEncodingStats::printTo(std::ostream& out) const { @@ -3805,9 +3917,9 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast65; - xfer += iprot->readI32(ecast65); - this->type = (Type::type)ecast65; + int32_t ecast67; + xfer += iprot->readI32(ecast67); + this->type = (Type::type)ecast67; isset_type = true; } else { xfer += iprot->skip(ftype); @@ -3817,16 +3929,16 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encodings.clear(); - uint32_t _size66; - ::apache::thrift::protocol::TType _etype69; - xfer += iprot->readListBegin(_etype69, _size66); - this->encodings.resize(_size66); - uint32_t _i70; - for (_i70 = 0; _i70 < _size66; ++_i70) + uint32_t _size68; + ::apache::thrift::protocol::TType _etype71; + xfer += iprot->readListBegin(_etype71, _size68); + this->encodings.resize(_size68); + uint32_t _i72; + for (_i72 = 0; _i72 < _size68; ++_i72) { - int32_t ecast71; - xfer += iprot->readI32(ecast71); - this->encodings[_i70] = (Encoding::type)ecast71; + int32_t ecast73; + xfer += iprot->readI32(ecast73); + this->encodings[_i72] = (Encoding::type)ecast73; } xfer += iprot->readListEnd(); } @@ -3839,14 +3951,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size72; - ::apache::thrift::protocol::TType _etype75; - xfer += iprot->readListBegin(_etype75, _size72); - this->path_in_schema.resize(_size72); - uint32_t _i76; - for (_i76 = 0; _i76 < _size72; ++_i76) + uint32_t _size74; + ::apache::thrift::protocol::TType _etype77; + xfer += iprot->readListBegin(_etype77, _size74); + this->path_in_schema.resize(_size74); + uint32_t _i78; + for (_i78 = 0; _i78 < _size74; ++_i78) { - xfer += iprot->readString(this->path_in_schema[_i76]); + xfer += iprot->readString(this->path_in_schema[_i78]); } xfer += iprot->readListEnd(); } @@ -3857,9 +3969,9 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast77; - xfer += iprot->readI32(ecast77); - this->codec = (CompressionCodec::type)ecast77; + int32_t ecast79; + xfer += iprot->readI32(ecast79); + this->codec = (CompressionCodec::type)ecast79; isset_codec = true; } else { xfer += iprot->skip(ftype); @@ -3893,14 +4005,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->key_value_metadata.clear(); - uint32_t _size78; - ::apache::thrift::protocol::TType _etype81; - xfer += iprot->readListBegin(_etype81, _size78); - this->key_value_metadata.resize(_size78); - uint32_t _i82; - for (_i82 = 0; _i82 < _size78; ++_i82) + uint32_t _size80; + ::apache::thrift::protocol::TType _etype83; + xfer += iprot->readListBegin(_etype83, _size80); + this->key_value_metadata.resize(_size80); + uint32_t _i84; + for (_i84 = 0; _i84 < _size80; ++_i84) { - xfer += this->key_value_metadata[_i82].read(iprot); + xfer += this->key_value_metadata[_i84].read(iprot); } xfer += iprot->readListEnd(); } @@ -3945,14 +4057,14 @@ uint32_t ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->encoding_stats.clear(); - uint32_t _size83; - ::apache::thrift::protocol::TType _etype86; - xfer += iprot->readListBegin(_etype86, _size83); - this->encoding_stats.resize(_size83); - uint32_t _i87; - for (_i87 = 0; _i87 < _size83; ++_i87) + uint32_t _size85; + ::apache::thrift::protocol::TType _etype88; + xfer += iprot->readListBegin(_etype88, _size85); + this->encoding_stats.resize(_size85); + uint32_t _i89; + for (_i89 = 0; _i89 < _size85; ++_i89) { - xfer += this->encoding_stats[_i87].read(iprot); + xfer += this->encoding_stats[_i89].read(iprot); } xfer += iprot->readListEnd(); } @@ -4001,10 +4113,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("encodings", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->encodings.size())); - std::vector ::const_iterator _iter88; - for (_iter88 = this->encodings.begin(); _iter88 != this->encodings.end(); ++_iter88) + std::vector ::const_iterator _iter90; + for (_iter90 = this->encodings.begin(); _iter90 != this->encodings.end(); ++_iter90) { - xfer += oprot->writeI32((int32_t)(*_iter88)); + xfer += oprot->writeI32((int32_t)(*_iter90)); } xfer += oprot->writeListEnd(); } @@ -4013,10 +4125,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter89; - for (_iter89 = this->path_in_schema.begin(); _iter89 != this->path_in_schema.end(); ++_iter89) + std::vector ::const_iterator _iter91; + for (_iter91 = this->path_in_schema.begin(); _iter91 != this->path_in_schema.end(); ++_iter91) { - xfer += oprot->writeString((*_iter89)); + xfer += oprot->writeString((*_iter91)); } xfer += oprot->writeListEnd(); } @@ -4042,10 +4154,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 8); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter90; - for (_iter90 = this->key_value_metadata.begin(); _iter90 != this->key_value_metadata.end(); ++_iter90) + std::vector ::const_iterator _iter92; + for (_iter92 = this->key_value_metadata.begin(); _iter92 != this->key_value_metadata.end(); ++_iter92) { - xfer += (*_iter90).write(oprot); + xfer += (*_iter92).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4074,10 +4186,10 @@ uint32_t ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeFieldBegin("encoding_stats", ::apache::thrift::protocol::T_LIST, 13); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->encoding_stats.size())); - std::vector ::const_iterator _iter91; - for (_iter91 = this->encoding_stats.begin(); _iter91 != this->encoding_stats.end(); ++_iter91) + std::vector ::const_iterator _iter93; + for (_iter93 = this->encoding_stats.begin(); _iter93 != this->encoding_stats.end(); ++_iter93) { - xfer += (*_iter91).write(oprot); + xfer += (*_iter93).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4106,37 +4218,37 @@ void swap(ColumnMetaData &a, ColumnMetaData &b) { swap(a.__isset, b.__isset); } -ColumnMetaData::ColumnMetaData(const ColumnMetaData& other92) { - type = other92.type; - encodings = other92.encodings; - path_in_schema = other92.path_in_schema; - codec = other92.codec; - num_values = other92.num_values; - total_uncompressed_size = other92.total_uncompressed_size; - total_compressed_size = other92.total_compressed_size; - key_value_metadata = other92.key_value_metadata; - data_page_offset = other92.data_page_offset; - index_page_offset = other92.index_page_offset; - dictionary_page_offset = other92.dictionary_page_offset; - statistics = other92.statistics; - encoding_stats = other92.encoding_stats; - __isset = other92.__isset; +ColumnMetaData::ColumnMetaData(const ColumnMetaData& other94) { + type = other94.type; + encodings = other94.encodings; + path_in_schema = other94.path_in_schema; + codec = other94.codec; + num_values = other94.num_values; + total_uncompressed_size = other94.total_uncompressed_size; + total_compressed_size = other94.total_compressed_size; + key_value_metadata = other94.key_value_metadata; + data_page_offset = other94.data_page_offset; + index_page_offset = other94.index_page_offset; + dictionary_page_offset = other94.dictionary_page_offset; + statistics = other94.statistics; + encoding_stats = other94.encoding_stats; + __isset = other94.__isset; } -ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other93) { - type = other93.type; - encodings = other93.encodings; - path_in_schema = other93.path_in_schema; - codec = other93.codec; - num_values = other93.num_values; - total_uncompressed_size = other93.total_uncompressed_size; - total_compressed_size = other93.total_compressed_size; - key_value_metadata = other93.key_value_metadata; - data_page_offset = other93.data_page_offset; - index_page_offset = other93.index_page_offset; - dictionary_page_offset = other93.dictionary_page_offset; - statistics = other93.statistics; - encoding_stats = other93.encoding_stats; - __isset = other93.__isset; +ColumnMetaData& ColumnMetaData::operator=(const ColumnMetaData& other95) { + type = other95.type; + encodings = other95.encodings; + path_in_schema = other95.path_in_schema; + codec = other95.codec; + num_values = other95.num_values; + total_uncompressed_size = other95.total_uncompressed_size; + total_compressed_size = other95.total_compressed_size; + key_value_metadata = other95.key_value_metadata; + data_page_offset = other95.data_page_offset; + index_page_offset = other95.index_page_offset; + dictionary_page_offset = other95.dictionary_page_offset; + statistics = other95.statistics; + encoding_stats = other95.encoding_stats; + __isset = other95.__isset; return *this; } void ColumnMetaData::printTo(std::ostream& out) const { @@ -4213,11 +4325,11 @@ void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b) { (void) b; } -EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other94) { - (void) other94; +EncryptionWithFooterKey::EncryptionWithFooterKey(const EncryptionWithFooterKey& other96) { + (void) other96; } -EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other95) { - (void) other95; +EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const EncryptionWithFooterKey& other97) { + (void) other97; return *this; } void EncryptionWithFooterKey::printTo(std::ostream& out) const { @@ -4235,9 +4347,9 @@ void EncryptionWithColumnKey::__set_path_in_schema(const std::vectorpath_in_schema = val; } -void EncryptionWithColumnKey::__set_column_key_metadata(const std::string& val) { - this->column_key_metadata = val; -__isset.column_key_metadata = true; +void EncryptionWithColumnKey::__set_key_metadata(const std::string& val) { + this->key_metadata = val; +__isset.key_metadata = true; } std::ostream& operator<<(std::ostream& out, const EncryptionWithColumnKey& obj) { @@ -4272,14 +4384,14 @@ uint32_t EncryptionWithColumnKey::read(::apache::thrift::protocol::TProtocol* ip if (ftype == ::apache::thrift::protocol::T_LIST) { { this->path_in_schema.clear(); - uint32_t _size96; - ::apache::thrift::protocol::TType _etype99; - xfer += iprot->readListBegin(_etype99, _size96); - this->path_in_schema.resize(_size96); - uint32_t _i100; - for (_i100 = 0; _i100 < _size96; ++_i100) + uint32_t _size98; + ::apache::thrift::protocol::TType _etype101; + xfer += iprot->readListBegin(_etype101, _size98); + this->path_in_schema.resize(_size98); + uint32_t _i102; + for (_i102 = 0; _i102 < _size98; ++_i102) { - xfer += iprot->readString(this->path_in_schema[_i100]); + xfer += iprot->readString(this->path_in_schema[_i102]); } xfer += iprot->readListEnd(); } @@ -4290,8 +4402,8 @@ uint32_t EncryptionWithColumnKey::read(::apache::thrift::protocol::TProtocol* ip break; case 2: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->column_key_metadata); - this->__isset.column_key_metadata = true; + xfer += iprot->readBinary(this->key_metadata); + this->__isset.key_metadata = true; } else { xfer += iprot->skip(ftype); } @@ -4318,18 +4430,18 @@ uint32_t EncryptionWithColumnKey::write(::apache::thrift::protocol::TProtocol* o xfer += oprot->writeFieldBegin("path_in_schema", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->path_in_schema.size())); - std::vector ::const_iterator _iter101; - for (_iter101 = this->path_in_schema.begin(); _iter101 != this->path_in_schema.end(); ++_iter101) + std::vector ::const_iterator _iter103; + for (_iter103 = this->path_in_schema.begin(); _iter103 != this->path_in_schema.end(); ++_iter103) { - xfer += oprot->writeString((*_iter101)); + xfer += oprot->writeString((*_iter103)); } xfer += oprot->writeListEnd(); } xfer += oprot->writeFieldEnd(); - if (this->__isset.column_key_metadata) { - xfer += oprot->writeFieldBegin("column_key_metadata", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->column_key_metadata); + if (this->__isset.key_metadata) { + xfer += oprot->writeFieldBegin("key_metadata", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->key_metadata); xfer += oprot->writeFieldEnd(); } xfer += oprot->writeFieldStop(); @@ -4340,26 +4452,26 @@ uint32_t EncryptionWithColumnKey::write(::apache::thrift::protocol::TProtocol* o void swap(EncryptionWithColumnKey &a, EncryptionWithColumnKey &b) { using ::std::swap; swap(a.path_in_schema, b.path_in_schema); - swap(a.column_key_metadata, b.column_key_metadata); + swap(a.key_metadata, b.key_metadata); swap(a.__isset, b.__isset); } -EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other102) { - path_in_schema = other102.path_in_schema; - column_key_metadata = other102.column_key_metadata; - __isset = other102.__isset; +EncryptionWithColumnKey::EncryptionWithColumnKey(const EncryptionWithColumnKey& other104) { + path_in_schema = other104.path_in_schema; + key_metadata = other104.key_metadata; + __isset = other104.__isset; } -EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other103) { - path_in_schema = other103.path_in_schema; - column_key_metadata = other103.column_key_metadata; - __isset = other103.__isset; +EncryptionWithColumnKey& EncryptionWithColumnKey::operator=(const EncryptionWithColumnKey& other105) { + path_in_schema = other105.path_in_schema; + key_metadata = other105.key_metadata; + __isset = other105.__isset; return *this; } void EncryptionWithColumnKey::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "EncryptionWithColumnKey("; out << "path_in_schema=" << to_string(path_in_schema); - out << ", " << "column_key_metadata="; (__isset.column_key_metadata ? (out << to_string(column_key_metadata)) : (out << "")); + out << ", " << "key_metadata="; (__isset.key_metadata ? (out << to_string(key_metadata)) : (out << "")); out << ")"; } @@ -4460,15 +4572,15 @@ void swap(ColumnCryptoMetaData &a, ColumnCryptoMetaData &b) { swap(a.__isset, b.__isset); } -ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other104) { - ENCRYPTION_WITH_FOOTER_KEY = other104.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other104.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other104.__isset; +ColumnCryptoMetaData::ColumnCryptoMetaData(const ColumnCryptoMetaData& other106) { + ENCRYPTION_WITH_FOOTER_KEY = other106.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other106.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other106.__isset; } -ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other105) { - ENCRYPTION_WITH_FOOTER_KEY = other105.ENCRYPTION_WITH_FOOTER_KEY; - ENCRYPTION_WITH_COLUMN_KEY = other105.ENCRYPTION_WITH_COLUMN_KEY; - __isset = other105.__isset; +ColumnCryptoMetaData& ColumnCryptoMetaData::operator=(const ColumnCryptoMetaData& other107) { + ENCRYPTION_WITH_FOOTER_KEY = other107.ENCRYPTION_WITH_FOOTER_KEY; + ENCRYPTION_WITH_COLUMN_KEY = other107.ENCRYPTION_WITH_COLUMN_KEY; + __isset = other107.__isset; return *this; } void ColumnCryptoMetaData::printTo(std::ostream& out) const { @@ -4518,9 +4630,14 @@ void ColumnChunk::__set_column_index_length(const int32_t val) { __isset.column_index_length = true; } -void ColumnChunk::__set_crypto_meta_data(const ColumnCryptoMetaData& val) { - this->crypto_meta_data = val; -__isset.crypto_meta_data = true; +void ColumnChunk::__set_crypto_metadata(const ColumnCryptoMetaData& val) { + this->crypto_metadata = val; +__isset.crypto_metadata = true; +} + +void ColumnChunk::__set_encrypted_column_metadata(const std::string& val) { + this->encrypted_column_metadata = val; +__isset.encrypted_column_metadata = true; } std::ostream& operator<<(std::ostream& out, const ColumnChunk& obj) { @@ -4609,8 +4726,16 @@ uint32_t ColumnChunk::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 8: if (ftype == ::apache::thrift::protocol::T_STRUCT) { - xfer += this->crypto_meta_data.read(iprot); - this->__isset.crypto_meta_data = true; + xfer += this->crypto_metadata.read(iprot); + this->__isset.crypto_metadata = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 9: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->encrypted_column_metadata); + this->__isset.encrypted_column_metadata = true; } else { xfer += iprot->skip(ftype); } @@ -4668,9 +4793,14 @@ uint32_t ColumnChunk::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeI32(this->column_index_length); xfer += oprot->writeFieldEnd(); } - if (this->__isset.crypto_meta_data) { - xfer += oprot->writeFieldBegin("crypto_meta_data", ::apache::thrift::protocol::T_STRUCT, 8); - xfer += this->crypto_meta_data.write(oprot); + if (this->__isset.crypto_metadata) { + xfer += oprot->writeFieldBegin("crypto_metadata", ::apache::thrift::protocol::T_STRUCT, 8); + xfer += this->crypto_metadata.write(oprot); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.encrypted_column_metadata) { + xfer += oprot->writeFieldBegin("encrypted_column_metadata", ::apache::thrift::protocol::T_STRING, 9); + xfer += oprot->writeBinary(this->encrypted_column_metadata); xfer += oprot->writeFieldEnd(); } xfer += oprot->writeFieldStop(); @@ -4687,31 +4817,34 @@ void swap(ColumnChunk &a, ColumnChunk &b) { swap(a.offset_index_length, b.offset_index_length); swap(a.column_index_offset, b.column_index_offset); swap(a.column_index_length, b.column_index_length); - swap(a.crypto_meta_data, b.crypto_meta_data); + swap(a.crypto_metadata, b.crypto_metadata); + swap(a.encrypted_column_metadata, b.encrypted_column_metadata); swap(a.__isset, b.__isset); } -ColumnChunk::ColumnChunk(const ColumnChunk& other106) { - file_path = other106.file_path; - file_offset = other106.file_offset; - meta_data = other106.meta_data; - offset_index_offset = other106.offset_index_offset; - offset_index_length = other106.offset_index_length; - column_index_offset = other106.column_index_offset; - column_index_length = other106.column_index_length; - crypto_meta_data = other106.crypto_meta_data; - __isset = other106.__isset; +ColumnChunk::ColumnChunk(const ColumnChunk& other108) { + file_path = other108.file_path; + file_offset = other108.file_offset; + meta_data = other108.meta_data; + offset_index_offset = other108.offset_index_offset; + offset_index_length = other108.offset_index_length; + column_index_offset = other108.column_index_offset; + column_index_length = other108.column_index_length; + crypto_metadata = other108.crypto_metadata; + encrypted_column_metadata = other108.encrypted_column_metadata; + __isset = other108.__isset; } -ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other107) { - file_path = other107.file_path; - file_offset = other107.file_offset; - meta_data = other107.meta_data; - offset_index_offset = other107.offset_index_offset; - offset_index_length = other107.offset_index_length; - column_index_offset = other107.column_index_offset; - column_index_length = other107.column_index_length; - crypto_meta_data = other107.crypto_meta_data; - __isset = other107.__isset; +ColumnChunk& ColumnChunk::operator=(const ColumnChunk& other109) { + file_path = other109.file_path; + file_offset = other109.file_offset; + meta_data = other109.meta_data; + offset_index_offset = other109.offset_index_offset; + offset_index_length = other109.offset_index_length; + column_index_offset = other109.column_index_offset; + column_index_length = other109.column_index_length; + crypto_metadata = other109.crypto_metadata; + encrypted_column_metadata = other109.encrypted_column_metadata; + __isset = other109.__isset; return *this; } void ColumnChunk::printTo(std::ostream& out) const { @@ -4724,7 +4857,8 @@ void ColumnChunk::printTo(std::ostream& out) const { out << ", " << "offset_index_length="; (__isset.offset_index_length ? (out << to_string(offset_index_length)) : (out << "")); out << ", " << "column_index_offset="; (__isset.column_index_offset ? (out << to_string(column_index_offset)) : (out << "")); out << ", " << "column_index_length="; (__isset.column_index_length ? (out << to_string(column_index_length)) : (out << "")); - out << ", " << "crypto_meta_data="; (__isset.crypto_meta_data ? (out << to_string(crypto_meta_data)) : (out << "")); + out << ", " << "crypto_metadata="; (__isset.crypto_metadata ? (out << to_string(crypto_metadata)) : (out << "")); + out << ", " << "encrypted_column_metadata="; (__isset.encrypted_column_metadata ? (out << to_string(encrypted_column_metadata)) : (out << "")); out << ")"; } @@ -4759,6 +4893,11 @@ void RowGroup::__set_total_compressed_size(const int64_t val) { this->total_compressed_size = val; __isset.total_compressed_size = true; } + +void RowGroup::__set_ordinal(const int16_t val) { + this->ordinal = val; +__isset.ordinal = true; +} std::ostream& operator<<(std::ostream& out, const RowGroup& obj) { obj.printTo(out); @@ -4794,14 +4933,14 @@ uint32_t RowGroup::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->columns.clear(); - uint32_t _size108; - ::apache::thrift::protocol::TType _etype111; - xfer += iprot->readListBegin(_etype111, _size108); - this->columns.resize(_size108); - uint32_t _i112; - for (_i112 = 0; _i112 < _size108; ++_i112) + uint32_t _size110; + ::apache::thrift::protocol::TType _etype113; + xfer += iprot->readListBegin(_etype113, _size110); + this->columns.resize(_size110); + uint32_t _i114; + for (_i114 = 0; _i114 < _size110; ++_i114) { - xfer += this->columns[_i112].read(iprot); + xfer += this->columns[_i114].read(iprot); } xfer += iprot->readListEnd(); } @@ -4830,14 +4969,14 @@ uint32_t RowGroup::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->sorting_columns.clear(); - uint32_t _size113; - ::apache::thrift::protocol::TType _etype116; - xfer += iprot->readListBegin(_etype116, _size113); - this->sorting_columns.resize(_size113); - uint32_t _i117; - for (_i117 = 0; _i117 < _size113; ++_i117) + uint32_t _size115; + ::apache::thrift::protocol::TType _etype118; + xfer += iprot->readListBegin(_etype118, _size115); + this->sorting_columns.resize(_size115); + uint32_t _i119; + for (_i119 = 0; _i119 < _size115; ++_i119) { - xfer += this->sorting_columns[_i117].read(iprot); + xfer += this->sorting_columns[_i119].read(iprot); } xfer += iprot->readListEnd(); } @@ -4862,6 +5001,14 @@ uint32_t RowGroup::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 7: + if (ftype == ::apache::thrift::protocol::T_I16) { + xfer += iprot->readI16(this->ordinal); + this->__isset.ordinal = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -4888,10 +5035,10 @@ uint32_t RowGroup::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("columns", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->columns.size())); - std::vector ::const_iterator _iter118; - for (_iter118 = this->columns.begin(); _iter118 != this->columns.end(); ++_iter118) + std::vector ::const_iterator _iter120; + for (_iter120 = this->columns.begin(); _iter120 != this->columns.end(); ++_iter120) { - xfer += (*_iter118).write(oprot); + xfer += (*_iter120).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4909,10 +5056,10 @@ uint32_t RowGroup::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("sorting_columns", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sorting_columns.size())); - std::vector ::const_iterator _iter119; - for (_iter119 = this->sorting_columns.begin(); _iter119 != this->sorting_columns.end(); ++_iter119) + std::vector ::const_iterator _iter121; + for (_iter121 = this->sorting_columns.begin(); _iter121 != this->sorting_columns.end(); ++_iter121) { - xfer += (*_iter119).write(oprot); + xfer += (*_iter121).write(oprot); } xfer += oprot->writeListEnd(); } @@ -4928,6 +5075,11 @@ uint32_t RowGroup::write(::apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeI64(this->total_compressed_size); xfer += oprot->writeFieldEnd(); } + if (this->__isset.ordinal) { + xfer += oprot->writeFieldBegin("ordinal", ::apache::thrift::protocol::T_I16, 7); + xfer += oprot->writeI16(this->ordinal); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -4941,26 +5093,29 @@ void swap(RowGroup &a, RowGroup &b) { swap(a.sorting_columns, b.sorting_columns); swap(a.file_offset, b.file_offset); swap(a.total_compressed_size, b.total_compressed_size); + swap(a.ordinal, b.ordinal); swap(a.__isset, b.__isset); } -RowGroup::RowGroup(const RowGroup& other120) { - columns = other120.columns; - total_byte_size = other120.total_byte_size; - num_rows = other120.num_rows; - sorting_columns = other120.sorting_columns; - file_offset = other120.file_offset; - total_compressed_size = other120.total_compressed_size; - __isset = other120.__isset; +RowGroup::RowGroup(const RowGroup& other122) { + columns = other122.columns; + total_byte_size = other122.total_byte_size; + num_rows = other122.num_rows; + sorting_columns = other122.sorting_columns; + file_offset = other122.file_offset; + total_compressed_size = other122.total_compressed_size; + ordinal = other122.ordinal; + __isset = other122.__isset; } -RowGroup& RowGroup::operator=(const RowGroup& other121) { - columns = other121.columns; - total_byte_size = other121.total_byte_size; - num_rows = other121.num_rows; - sorting_columns = other121.sorting_columns; - file_offset = other121.file_offset; - total_compressed_size = other121.total_compressed_size; - __isset = other121.__isset; +RowGroup& RowGroup::operator=(const RowGroup& other123) { + columns = other123.columns; + total_byte_size = other123.total_byte_size; + num_rows = other123.num_rows; + sorting_columns = other123.sorting_columns; + file_offset = other123.file_offset; + total_compressed_size = other123.total_compressed_size; + ordinal = other123.ordinal; + __isset = other123.__isset; return *this; } void RowGroup::printTo(std::ostream& out) const { @@ -4972,6 +5127,7 @@ void RowGroup::printTo(std::ostream& out) const { out << ", " << "sorting_columns="; (__isset.sorting_columns ? (out << to_string(sorting_columns)) : (out << "")); out << ", " << "file_offset="; (__isset.file_offset ? (out << to_string(file_offset)) : (out << "")); out << ", " << "total_compressed_size="; (__isset.total_compressed_size ? (out << to_string(total_compressed_size)) : (out << "")); + out << ", " << "ordinal="; (__isset.ordinal ? (out << to_string(ordinal)) : (out << "")); out << ")"; } @@ -5030,11 +5186,11 @@ void swap(TypeDefinedOrder &a, TypeDefinedOrder &b) { (void) b; } -TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other122) { - (void) other122; +TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other124) { + (void) other124; } -TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other123) { - (void) other123; +TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder& other125) { + (void) other125; return *this; } void TypeDefinedOrder::printTo(std::ostream& out) const { @@ -5121,13 +5277,13 @@ void swap(ColumnOrder &a, ColumnOrder &b) { swap(a.__isset, b.__isset); } -ColumnOrder::ColumnOrder(const ColumnOrder& other124) { - TYPE_ORDER = other124.TYPE_ORDER; - __isset = other124.__isset; +ColumnOrder::ColumnOrder(const ColumnOrder& other126) { + TYPE_ORDER = other126.TYPE_ORDER; + __isset = other126.__isset; } -ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other125) { - TYPE_ORDER = other125.TYPE_ORDER; - __isset = other125.__isset; +ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other127) { + TYPE_ORDER = other127.TYPE_ORDER; + __isset = other127.__isset; return *this; } void ColumnOrder::printTo(std::ostream& out) const { @@ -5255,15 +5411,15 @@ void swap(PageLocation &a, PageLocation &b) { swap(a.first_row_index, b.first_row_index); } -PageLocation::PageLocation(const PageLocation& other126) { - offset = other126.offset; - compressed_page_size = other126.compressed_page_size; - first_row_index = other126.first_row_index; +PageLocation::PageLocation(const PageLocation& other128) { + offset = other128.offset; + compressed_page_size = other128.compressed_page_size; + first_row_index = other128.first_row_index; } -PageLocation& PageLocation::operator=(const PageLocation& other127) { - offset = other127.offset; - compressed_page_size = other127.compressed_page_size; - first_row_index = other127.first_row_index; +PageLocation& PageLocation::operator=(const PageLocation& other129) { + offset = other129.offset; + compressed_page_size = other129.compressed_page_size; + first_row_index = other129.first_row_index; return *this; } void PageLocation::printTo(std::ostream& out) const { @@ -5316,14 +5472,14 @@ uint32_t OffsetIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->page_locations.clear(); - uint32_t _size128; - ::apache::thrift::protocol::TType _etype131; - xfer += iprot->readListBegin(_etype131, _size128); - this->page_locations.resize(_size128); - uint32_t _i132; - for (_i132 = 0; _i132 < _size128; ++_i132) + uint32_t _size130; + ::apache::thrift::protocol::TType _etype133; + xfer += iprot->readListBegin(_etype133, _size130); + this->page_locations.resize(_size130); + uint32_t _i134; + for (_i134 = 0; _i134 < _size130; ++_i134) { - xfer += this->page_locations[_i132].read(iprot); + xfer += this->page_locations[_i134].read(iprot); } xfer += iprot->readListEnd(); } @@ -5354,10 +5510,10 @@ uint32_t OffsetIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("page_locations", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->page_locations.size())); - std::vector ::const_iterator _iter133; - for (_iter133 = this->page_locations.begin(); _iter133 != this->page_locations.end(); ++_iter133) + std::vector ::const_iterator _iter135; + for (_iter135 = this->page_locations.begin(); _iter135 != this->page_locations.end(); ++_iter135) { - xfer += (*_iter133).write(oprot); + xfer += (*_iter135).write(oprot); } xfer += oprot->writeListEnd(); } @@ -5373,11 +5529,11 @@ void swap(OffsetIndex &a, OffsetIndex &b) { swap(a.page_locations, b.page_locations); } -OffsetIndex::OffsetIndex(const OffsetIndex& other134) { - page_locations = other134.page_locations; +OffsetIndex::OffsetIndex(const OffsetIndex& other136) { + page_locations = other136.page_locations; } -OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other135) { - page_locations = other135.page_locations; +OffsetIndex& OffsetIndex::operator=(const OffsetIndex& other137) { + page_locations = other137.page_locations; return *this; } void OffsetIndex::printTo(std::ostream& out) const { @@ -5448,14 +5604,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_pages.clear(); - uint32_t _size136; - ::apache::thrift::protocol::TType _etype139; - xfer += iprot->readListBegin(_etype139, _size136); - this->null_pages.resize(_size136); - uint32_t _i140; - for (_i140 = 0; _i140 < _size136; ++_i140) + uint32_t _size138; + ::apache::thrift::protocol::TType _etype141; + xfer += iprot->readListBegin(_etype141, _size138); + this->null_pages.resize(_size138); + uint32_t _i142; + for (_i142 = 0; _i142 < _size138; ++_i142) { - xfer += iprot->readBool(this->null_pages[_i140]); + xfer += iprot->readBool(this->null_pages[_i142]); } xfer += iprot->readListEnd(); } @@ -5468,14 +5624,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->min_values.clear(); - uint32_t _size141; - ::apache::thrift::protocol::TType _etype144; - xfer += iprot->readListBegin(_etype144, _size141); - this->min_values.resize(_size141); - uint32_t _i145; - for (_i145 = 0; _i145 < _size141; ++_i145) + uint32_t _size143; + ::apache::thrift::protocol::TType _etype146; + xfer += iprot->readListBegin(_etype146, _size143); + this->min_values.resize(_size143); + uint32_t _i147; + for (_i147 = 0; _i147 < _size143; ++_i147) { - xfer += iprot->readBinary(this->min_values[_i145]); + xfer += iprot->readBinary(this->min_values[_i147]); } xfer += iprot->readListEnd(); } @@ -5488,14 +5644,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->max_values.clear(); - uint32_t _size146; - ::apache::thrift::protocol::TType _etype149; - xfer += iprot->readListBegin(_etype149, _size146); - this->max_values.resize(_size146); - uint32_t _i150; - for (_i150 = 0; _i150 < _size146; ++_i150) + uint32_t _size148; + ::apache::thrift::protocol::TType _etype151; + xfer += iprot->readListBegin(_etype151, _size148); + this->max_values.resize(_size148); + uint32_t _i152; + for (_i152 = 0; _i152 < _size148; ++_i152) { - xfer += iprot->readBinary(this->max_values[_i150]); + xfer += iprot->readBinary(this->max_values[_i152]); } xfer += iprot->readListEnd(); } @@ -5506,9 +5662,9 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 4: if (ftype == ::apache::thrift::protocol::T_I32) { - int32_t ecast151; - xfer += iprot->readI32(ecast151); - this->boundary_order = (BoundaryOrder::type)ecast151; + int32_t ecast153; + xfer += iprot->readI32(ecast153); + this->boundary_order = (BoundaryOrder::type)ecast153; isset_boundary_order = true; } else { xfer += iprot->skip(ftype); @@ -5518,14 +5674,14 @@ uint32_t ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) { if (ftype == ::apache::thrift::protocol::T_LIST) { { this->null_counts.clear(); - uint32_t _size152; - ::apache::thrift::protocol::TType _etype155; - xfer += iprot->readListBegin(_etype155, _size152); - this->null_counts.resize(_size152); - uint32_t _i156; - for (_i156 = 0; _i156 < _size152; ++_i156) + uint32_t _size154; + ::apache::thrift::protocol::TType _etype157; + xfer += iprot->readListBegin(_etype157, _size154); + this->null_counts.resize(_size154); + uint32_t _i158; + for (_i158 = 0; _i158 < _size154; ++_i158) { - xfer += iprot->readI64(this->null_counts[_i156]); + xfer += iprot->readI64(this->null_counts[_i158]); } xfer += iprot->readListEnd(); } @@ -5562,10 +5718,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("null_pages", ::apache::thrift::protocol::T_LIST, 1); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_BOOL, static_cast(this->null_pages.size())); - std::vector ::const_iterator _iter157; - for (_iter157 = this->null_pages.begin(); _iter157 != this->null_pages.end(); ++_iter157) + std::vector ::const_iterator _iter159; + for (_iter159 = this->null_pages.begin(); _iter159 != this->null_pages.end(); ++_iter159) { - xfer += oprot->writeBool((*_iter157)); + xfer += oprot->writeBool((*_iter159)); } xfer += oprot->writeListEnd(); } @@ -5574,10 +5730,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("min_values", ::apache::thrift::protocol::T_LIST, 2); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->min_values.size())); - std::vector ::const_iterator _iter158; - for (_iter158 = this->min_values.begin(); _iter158 != this->min_values.end(); ++_iter158) + std::vector ::const_iterator _iter160; + for (_iter160 = this->min_values.begin(); _iter160 != this->min_values.end(); ++_iter160) { - xfer += oprot->writeBinary((*_iter158)); + xfer += oprot->writeBinary((*_iter160)); } xfer += oprot->writeListEnd(); } @@ -5586,10 +5742,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("max_values", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->max_values.size())); - std::vector ::const_iterator _iter159; - for (_iter159 = this->max_values.begin(); _iter159 != this->max_values.end(); ++_iter159) + std::vector ::const_iterator _iter161; + for (_iter161 = this->max_values.begin(); _iter161 != this->max_values.end(); ++_iter161) { - xfer += oprot->writeBinary((*_iter159)); + xfer += oprot->writeBinary((*_iter161)); } xfer += oprot->writeListEnd(); } @@ -5603,10 +5759,10 @@ uint32_t ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeFieldBegin("null_counts", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->null_counts.size())); - std::vector ::const_iterator _iter160; - for (_iter160 = this->null_counts.begin(); _iter160 != this->null_counts.end(); ++_iter160) + std::vector ::const_iterator _iter162; + for (_iter162 = this->null_counts.begin(); _iter162 != this->null_counts.end(); ++_iter162) { - xfer += oprot->writeI64((*_iter160)); + xfer += oprot->writeI64((*_iter162)); } xfer += oprot->writeListEnd(); } @@ -5627,21 +5783,21 @@ void swap(ColumnIndex &a, ColumnIndex &b) { swap(a.__isset, b.__isset); } -ColumnIndex::ColumnIndex(const ColumnIndex& other161) { - null_pages = other161.null_pages; - min_values = other161.min_values; - max_values = other161.max_values; - boundary_order = other161.boundary_order; - null_counts = other161.null_counts; - __isset = other161.__isset; +ColumnIndex::ColumnIndex(const ColumnIndex& other163) { + null_pages = other163.null_pages; + min_values = other163.min_values; + max_values = other163.max_values; + boundary_order = other163.boundary_order; + null_counts = other163.null_counts; + __isset = other163.__isset; } -ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other162) { - null_pages = other162.null_pages; - min_values = other162.min_values; - max_values = other162.max_values; - boundary_order = other162.boundary_order; - null_counts = other162.null_counts; - __isset = other162.__isset; +ColumnIndex& ColumnIndex::operator=(const ColumnIndex& other164) { + null_pages = other164.null_pages; + min_values = other164.min_values; + max_values = other164.max_values; + boundary_order = other164.boundary_order; + null_counts = other164.null_counts; + __isset = other164.__isset; return *this; } void ColumnIndex::printTo(std::ostream& out) const { @@ -5656,328 +5812,23 @@ void ColumnIndex::printTo(std::ostream& out) const { } -FileMetaData::~FileMetaData() throw() { -} - - -void FileMetaData::__set_version(const int32_t val) { - this->version = val; -} - -void FileMetaData::__set_schema(const std::vector & val) { - this->schema = val; -} - -void FileMetaData::__set_num_rows(const int64_t val) { - this->num_rows = val; -} - -void FileMetaData::__set_row_groups(const std::vector & val) { - this->row_groups = val; -} - -void FileMetaData::__set_key_value_metadata(const std::vector & val) { - this->key_value_metadata = val; -__isset.key_value_metadata = true; -} - -void FileMetaData::__set_created_by(const std::string& val) { - this->created_by = val; -__isset.created_by = true; -} - -void FileMetaData::__set_column_orders(const std::vector & val) { - this->column_orders = val; -__isset.column_orders = true; -} -std::ostream& operator<<(std::ostream& out, const FileMetaData& obj) -{ - obj.printTo(out); - return out; -} - - -uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { - - ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); - uint32_t xfer = 0; - std::string fname; - ::apache::thrift::protocol::TType ftype; - int16_t fid; - - xfer += iprot->readStructBegin(fname); - - using ::apache::thrift::protocol::TProtocolException; - - bool isset_version = false; - bool isset_schema = false; - bool isset_num_rows = false; - bool isset_row_groups = false; - - while (true) - { - xfer += iprot->readFieldBegin(fname, ftype, fid); - if (ftype == ::apache::thrift::protocol::T_STOP) { - break; - } - switch (fid) - { - case 1: - if (ftype == ::apache::thrift::protocol::T_I32) { - xfer += iprot->readI32(this->version); - isset_version = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 2: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->schema.clear(); - uint32_t _size163; - ::apache::thrift::protocol::TType _etype166; - xfer += iprot->readListBegin(_etype166, _size163); - this->schema.resize(_size163); - uint32_t _i167; - for (_i167 = 0; _i167 < _size163; ++_i167) - { - xfer += this->schema[_i167].read(iprot); - } - xfer += iprot->readListEnd(); - } - isset_schema = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->num_rows); - isset_num_rows = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 4: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->row_groups.clear(); - uint32_t _size168; - ::apache::thrift::protocol::TType _etype171; - xfer += iprot->readListBegin(_etype171, _size168); - this->row_groups.resize(_size168); - uint32_t _i172; - for (_i172 = 0; _i172 < _size168; ++_i172) - { - xfer += this->row_groups[_i172].read(iprot); - } - xfer += iprot->readListEnd(); - } - isset_row_groups = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 5: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->key_value_metadata.clear(); - uint32_t _size173; - ::apache::thrift::protocol::TType _etype176; - xfer += iprot->readListBegin(_etype176, _size173); - this->key_value_metadata.resize(_size173); - uint32_t _i177; - for (_i177 = 0; _i177 < _size173; ++_i177) - { - xfer += this->key_value_metadata[_i177].read(iprot); - } - xfer += iprot->readListEnd(); - } - this->__isset.key_value_metadata = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 6: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readString(this->created_by); - this->__isset.created_by = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 7: - if (ftype == ::apache::thrift::protocol::T_LIST) { - { - this->column_orders.clear(); - uint32_t _size178; - ::apache::thrift::protocol::TType _etype181; - xfer += iprot->readListBegin(_etype181, _size178); - this->column_orders.resize(_size178); - uint32_t _i182; - for (_i182 = 0; _i182 < _size178; ++_i182) - { - xfer += this->column_orders[_i182].read(iprot); - } - xfer += iprot->readListEnd(); - } - this->__isset.column_orders = true; - } else { - xfer += iprot->skip(ftype); - } - break; - default: - xfer += iprot->skip(ftype); - break; - } - xfer += iprot->readFieldEnd(); - } - - xfer += iprot->readStructEnd(); - - if (!isset_version) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_schema) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_num_rows) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_row_groups) - throw TProtocolException(TProtocolException::INVALID_DATA); - return xfer; -} - -uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const { - uint32_t xfer = 0; - ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); - xfer += oprot->writeStructBegin("FileMetaData"); - - xfer += oprot->writeFieldBegin("version", ::apache::thrift::protocol::T_I32, 1); - xfer += oprot->writeI32(this->version); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); - std::vector ::const_iterator _iter183; - for (_iter183 = this->schema.begin(); _iter183 != this->schema.end(); ++_iter183) - { - xfer += (*_iter183).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("num_rows", ::apache::thrift::protocol::T_I64, 3); - xfer += oprot->writeI64(this->num_rows); - xfer += oprot->writeFieldEnd(); - - xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); - std::vector ::const_iterator _iter184; - for (_iter184 = this->row_groups.begin(); _iter184 != this->row_groups.end(); ++_iter184) - { - xfer += (*_iter184).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - - if (this->__isset.key_value_metadata) { - xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); - std::vector ::const_iterator _iter185; - for (_iter185 = this->key_value_metadata.begin(); _iter185 != this->key_value_metadata.end(); ++_iter185) - { - xfer += (*_iter185).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.created_by) { - xfer += oprot->writeFieldBegin("created_by", ::apache::thrift::protocol::T_STRING, 6); - xfer += oprot->writeString(this->created_by); - xfer += oprot->writeFieldEnd(); - } - if (this->__isset.column_orders) { - xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); - { - xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); - std::vector ::const_iterator _iter186; - for (_iter186 = this->column_orders.begin(); _iter186 != this->column_orders.end(); ++_iter186) - { - xfer += (*_iter186).write(oprot); - } - xfer += oprot->writeListEnd(); - } - xfer += oprot->writeFieldEnd(); - } - xfer += oprot->writeFieldStop(); - xfer += oprot->writeStructEnd(); - return xfer; -} - -void swap(FileMetaData &a, FileMetaData &b) { - using ::std::swap; - swap(a.version, b.version); - swap(a.schema, b.schema); - swap(a.num_rows, b.num_rows); - swap(a.row_groups, b.row_groups); - swap(a.key_value_metadata, b.key_value_metadata); - swap(a.created_by, b.created_by); - swap(a.column_orders, b.column_orders); - swap(a.__isset, b.__isset); -} - -FileMetaData::FileMetaData(const FileMetaData& other187) { - version = other187.version; - schema = other187.schema; - num_rows = other187.num_rows; - row_groups = other187.row_groups; - key_value_metadata = other187.key_value_metadata; - created_by = other187.created_by; - column_orders = other187.column_orders; - __isset = other187.__isset; -} -FileMetaData& FileMetaData::operator=(const FileMetaData& other188) { - version = other188.version; - schema = other188.schema; - num_rows = other188.num_rows; - row_groups = other188.row_groups; - key_value_metadata = other188.key_value_metadata; - created_by = other188.created_by; - column_orders = other188.column_orders; - __isset = other188.__isset; - return *this; -} -void FileMetaData::printTo(std::ostream& out) const { - using ::apache::thrift::to_string; - out << "FileMetaData("; - out << "version=" << to_string(version); - out << ", " << "schema=" << to_string(schema); - out << ", " << "num_rows=" << to_string(num_rows); - out << ", " << "row_groups=" << to_string(row_groups); - out << ", " << "key_value_metadata="; (__isset.key_value_metadata ? (out << to_string(key_value_metadata)) : (out << "")); - out << ", " << "created_by="; (__isset.created_by ? (out << to_string(created_by)) : (out << "")); - out << ", " << "column_orders="; (__isset.column_orders ? (out << to_string(column_orders)) : (out << "")); - out << ")"; -} - - AesGcmV1::~AesGcmV1() throw() { } -void AesGcmV1::__set_aad_metadata(const std::string& val) { - this->aad_metadata = val; -__isset.aad_metadata = true; +void AesGcmV1::__set_aad_prefix(const std::string& val) { + this->aad_prefix = val; +__isset.aad_prefix = true; } -void AesGcmV1::__set_iv_prefix(const std::string& val) { - this->iv_prefix = val; -__isset.iv_prefix = true; +void AesGcmV1::__set_aad_file_unique(const std::string& val) { + this->aad_file_unique = val; +__isset.aad_file_unique = true; +} + +void AesGcmV1::__set_supply_aad_prefix(const bool val) { + this->supply_aad_prefix = val; +__isset.supply_aad_prefix = true; } std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj) { @@ -6009,16 +5860,24 @@ uint32_t AesGcmV1::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_metadata); - this->__isset.aad_metadata = true; + xfer += iprot->readBinary(this->aad_prefix); + this->__isset.aad_prefix = true; } else { xfer += iprot->skip(ftype); } break; case 2: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->iv_prefix); - this->__isset.iv_prefix = true; + xfer += iprot->readBinary(this->aad_file_unique); + this->__isset.aad_file_unique = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->supply_aad_prefix); + this->__isset.supply_aad_prefix = true; } else { xfer += iprot->skip(ftype); } @@ -6040,14 +5899,19 @@ uint32_t AesGcmV1::write(::apache::thrift::protocol::TProtocol* oprot) const { ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("AesGcmV1"); - if (this->__isset.aad_metadata) { - xfer += oprot->writeFieldBegin("aad_metadata", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeBinary(this->aad_metadata); + if (this->__isset.aad_prefix) { + xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->aad_prefix); xfer += oprot->writeFieldEnd(); } - if (this->__isset.iv_prefix) { - xfer += oprot->writeFieldBegin("iv_prefix", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->iv_prefix); + if (this->__isset.aad_file_unique) { + xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->aad_file_unique); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.supply_aad_prefix) { + xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); + xfer += oprot->writeBool(this->supply_aad_prefix); xfer += oprot->writeFieldEnd(); } xfer += oprot->writeFieldStop(); @@ -6057,27 +5921,31 @@ uint32_t AesGcmV1::write(::apache::thrift::protocol::TProtocol* oprot) const { void swap(AesGcmV1 &a, AesGcmV1 &b) { using ::std::swap; - swap(a.aad_metadata, b.aad_metadata); - swap(a.iv_prefix, b.iv_prefix); + swap(a.aad_prefix, b.aad_prefix); + swap(a.aad_file_unique, b.aad_file_unique); + swap(a.supply_aad_prefix, b.supply_aad_prefix); swap(a.__isset, b.__isset); } -AesGcmV1::AesGcmV1(const AesGcmV1& other189) { - aad_metadata = other189.aad_metadata; - iv_prefix = other189.iv_prefix; - __isset = other189.__isset; +AesGcmV1::AesGcmV1(const AesGcmV1& other165) { + aad_prefix = other165.aad_prefix; + aad_file_unique = other165.aad_file_unique; + supply_aad_prefix = other165.supply_aad_prefix; + __isset = other165.__isset; } -AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other190) { - aad_metadata = other190.aad_metadata; - iv_prefix = other190.iv_prefix; - __isset = other190.__isset; +AesGcmV1& AesGcmV1::operator=(const AesGcmV1& other166) { + aad_prefix = other166.aad_prefix; + aad_file_unique = other166.aad_file_unique; + supply_aad_prefix = other166.supply_aad_prefix; + __isset = other166.__isset; return *this; } void AesGcmV1::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "AesGcmV1("; - out << "aad_metadata="; (__isset.aad_metadata ? (out << to_string(aad_metadata)) : (out << "")); - out << ", " << "iv_prefix="; (__isset.iv_prefix ? (out << to_string(iv_prefix)) : (out << "")); + out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); out << ")"; } @@ -6086,19 +5954,19 @@ AesGcmCtrV1::~AesGcmCtrV1() throw() { } -void AesGcmCtrV1::__set_aad_metadata(const std::string& val) { - this->aad_metadata = val; -__isset.aad_metadata = true; +void AesGcmCtrV1::__set_aad_prefix(const std::string& val) { + this->aad_prefix = val; +__isset.aad_prefix = true; } -void AesGcmCtrV1::__set_gcm_iv_prefix(const std::string& val) { - this->gcm_iv_prefix = val; -__isset.gcm_iv_prefix = true; +void AesGcmCtrV1::__set_aad_file_unique(const std::string& val) { + this->aad_file_unique = val; +__isset.aad_file_unique = true; } -void AesGcmCtrV1::__set_ctr_iv_prefix(const std::string& val) { - this->ctr_iv_prefix = val; -__isset.ctr_iv_prefix = true; +void AesGcmCtrV1::__set_supply_aad_prefix(const bool val) { + this->supply_aad_prefix = val; +__isset.supply_aad_prefix = true; } std::ostream& operator<<(std::ostream& out, const AesGcmCtrV1& obj) { @@ -6130,24 +5998,24 @@ uint32_t AesGcmCtrV1::read(::apache::thrift::protocol::TProtocol* iprot) { { case 1: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->aad_metadata); - this->__isset.aad_metadata = true; + xfer += iprot->readBinary(this->aad_prefix); + this->__isset.aad_prefix = true; } else { xfer += iprot->skip(ftype); } break; case 2: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->gcm_iv_prefix); - this->__isset.gcm_iv_prefix = true; + xfer += iprot->readBinary(this->aad_file_unique); + this->__isset.aad_file_unique = true; } else { xfer += iprot->skip(ftype); } break; case 3: - if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->ctr_iv_prefix); - this->__isset.ctr_iv_prefix = true; + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->supply_aad_prefix); + this->__isset.supply_aad_prefix = true; } else { xfer += iprot->skip(ftype); } @@ -6169,19 +6037,19 @@ uint32_t AesGcmCtrV1::write(::apache::thrift::protocol::TProtocol* oprot) const ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("AesGcmCtrV1"); - if (this->__isset.aad_metadata) { - xfer += oprot->writeFieldBegin("aad_metadata", ::apache::thrift::protocol::T_STRING, 1); - xfer += oprot->writeBinary(this->aad_metadata); + if (this->__isset.aad_prefix) { + xfer += oprot->writeFieldBegin("aad_prefix", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->aad_prefix); xfer += oprot->writeFieldEnd(); } - if (this->__isset.gcm_iv_prefix) { - xfer += oprot->writeFieldBegin("gcm_iv_prefix", ::apache::thrift::protocol::T_STRING, 2); - xfer += oprot->writeBinary(this->gcm_iv_prefix); + if (this->__isset.aad_file_unique) { + xfer += oprot->writeFieldBegin("aad_file_unique", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->aad_file_unique); xfer += oprot->writeFieldEnd(); } - if (this->__isset.ctr_iv_prefix) { - xfer += oprot->writeFieldBegin("ctr_iv_prefix", ::apache::thrift::protocol::T_STRING, 3); - xfer += oprot->writeBinary(this->ctr_iv_prefix); + if (this->__isset.supply_aad_prefix) { + xfer += oprot->writeFieldBegin("supply_aad_prefix", ::apache::thrift::protocol::T_BOOL, 3); + xfer += oprot->writeBool(this->supply_aad_prefix); xfer += oprot->writeFieldEnd(); } xfer += oprot->writeFieldStop(); @@ -6191,31 +6059,31 @@ uint32_t AesGcmCtrV1::write(::apache::thrift::protocol::TProtocol* oprot) const void swap(AesGcmCtrV1 &a, AesGcmCtrV1 &b) { using ::std::swap; - swap(a.aad_metadata, b.aad_metadata); - swap(a.gcm_iv_prefix, b.gcm_iv_prefix); - swap(a.ctr_iv_prefix, b.ctr_iv_prefix); + swap(a.aad_prefix, b.aad_prefix); + swap(a.aad_file_unique, b.aad_file_unique); + swap(a.supply_aad_prefix, b.supply_aad_prefix); swap(a.__isset, b.__isset); } -AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other191) { - aad_metadata = other191.aad_metadata; - gcm_iv_prefix = other191.gcm_iv_prefix; - ctr_iv_prefix = other191.ctr_iv_prefix; - __isset = other191.__isset; +AesGcmCtrV1::AesGcmCtrV1(const AesGcmCtrV1& other167) { + aad_prefix = other167.aad_prefix; + aad_file_unique = other167.aad_file_unique; + supply_aad_prefix = other167.supply_aad_prefix; + __isset = other167.__isset; } -AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other192) { - aad_metadata = other192.aad_metadata; - gcm_iv_prefix = other192.gcm_iv_prefix; - ctr_iv_prefix = other192.ctr_iv_prefix; - __isset = other192.__isset; +AesGcmCtrV1& AesGcmCtrV1::operator=(const AesGcmCtrV1& other168) { + aad_prefix = other168.aad_prefix; + aad_file_unique = other168.aad_file_unique; + supply_aad_prefix = other168.supply_aad_prefix; + __isset = other168.__isset; return *this; } void AesGcmCtrV1::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "AesGcmCtrV1("; - out << "aad_metadata="; (__isset.aad_metadata ? (out << to_string(aad_metadata)) : (out << "")); - out << ", " << "gcm_iv_prefix="; (__isset.gcm_iv_prefix ? (out << to_string(gcm_iv_prefix)) : (out << "")); - out << ", " << "ctr_iv_prefix="; (__isset.ctr_iv_prefix ? (out << to_string(ctr_iv_prefix)) : (out << "")); + out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); out << ")"; } @@ -6316,15 +6184,15 @@ void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b) { swap(a.__isset, b.__isset); } -EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other193) { - AES_GCM_V1 = other193.AES_GCM_V1; - AES_GCM_CTR_V1 = other193.AES_GCM_CTR_V1; - __isset = other193.__isset; +EncryptionAlgorithm::EncryptionAlgorithm(const EncryptionAlgorithm& other169) { + AES_GCM_V1 = other169.AES_GCM_V1; + AES_GCM_CTR_V1 = other169.AES_GCM_CTR_V1; + __isset = other169.__isset; } -EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other194) { - AES_GCM_V1 = other194.AES_GCM_V1; - AES_GCM_CTR_V1 = other194.AES_GCM_CTR_V1; - __isset = other194.__isset; +EncryptionAlgorithm& EncryptionAlgorithm::operator=(const EncryptionAlgorithm& other170) { + AES_GCM_V1 = other170.AES_GCM_V1; + AES_GCM_CTR_V1 = other170.AES_GCM_CTR_V1; + __isset = other170.__isset; return *this; } void EncryptionAlgorithm::printTo(std::ostream& out) const { @@ -6336,6 +6204,360 @@ void EncryptionAlgorithm::printTo(std::ostream& out) const { } +FileMetaData::~FileMetaData() throw() { +} + + +void FileMetaData::__set_version(const int32_t val) { + this->version = val; +} + +void FileMetaData::__set_schema(const std::vector & val) { + this->schema = val; +} + +void FileMetaData::__set_num_rows(const int64_t val) { + this->num_rows = val; +} + +void FileMetaData::__set_row_groups(const std::vector & val) { + this->row_groups = val; +} + +void FileMetaData::__set_key_value_metadata(const std::vector & val) { + this->key_value_metadata = val; +__isset.key_value_metadata = true; +} + +void FileMetaData::__set_created_by(const std::string& val) { + this->created_by = val; +__isset.created_by = true; +} + +void FileMetaData::__set_column_orders(const std::vector & val) { + this->column_orders = val; +__isset.column_orders = true; +} + +void FileMetaData::__set_encryption_algorithm(const EncryptionAlgorithm& val) { + this->encryption_algorithm = val; +__isset.encryption_algorithm = true; +} + +void FileMetaData::__set_footer_signing_key_metadata(const std::string& val) { + this->footer_signing_key_metadata = val; +__isset.footer_signing_key_metadata = true; +} +std::ostream& operator<<(std::ostream& out, const FileMetaData& obj) +{ + obj.printTo(out); + return out; +} + + +uint32_t FileMetaData::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_version = false; + bool isset_schema = false; + bool isset_num_rows = false; + bool isset_row_groups = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->version); + isset_version = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->schema.clear(); + uint32_t _size171; + ::apache::thrift::protocol::TType _etype174; + xfer += iprot->readListBegin(_etype174, _size171); + this->schema.resize(_size171); + uint32_t _i175; + for (_i175 = 0; _i175 < _size171; ++_i175) + { + xfer += this->schema[_i175].read(iprot); + } + xfer += iprot->readListEnd(); + } + isset_schema = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->num_rows); + isset_num_rows = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->row_groups.clear(); + uint32_t _size176; + ::apache::thrift::protocol::TType _etype179; + xfer += iprot->readListBegin(_etype179, _size176); + this->row_groups.resize(_size176); + uint32_t _i180; + for (_i180 = 0; _i180 < _size176; ++_i180) + { + xfer += this->row_groups[_i180].read(iprot); + } + xfer += iprot->readListEnd(); + } + isset_row_groups = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->key_value_metadata.clear(); + uint32_t _size181; + ::apache::thrift::protocol::TType _etype184; + xfer += iprot->readListBegin(_etype184, _size181); + this->key_value_metadata.resize(_size181); + uint32_t _i185; + for (_i185 = 0; _i185 < _size181; ++_i185) + { + xfer += this->key_value_metadata[_i185].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.key_value_metadata = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 6: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->created_by); + this->__isset.created_by = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 7: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->column_orders.clear(); + uint32_t _size186; + ::apache::thrift::protocol::TType _etype189; + xfer += iprot->readListBegin(_etype189, _size186); + this->column_orders.resize(_size186); + uint32_t _i190; + for (_i190 = 0; _i190 < _size186; ++_i190) + { + xfer += this->column_orders[_i190].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.column_orders = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 8: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->encryption_algorithm.read(iprot); + this->__isset.encryption_algorithm = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 9: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->footer_signing_key_metadata); + this->__isset.footer_signing_key_metadata = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + if (!isset_version) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_schema) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_num_rows) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_row_groups) + throw TProtocolException(TProtocolException::INVALID_DATA); + return xfer; +} + +uint32_t FileMetaData::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("FileMetaData"); + + xfer += oprot->writeFieldBegin("version", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->version); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("schema", ::apache::thrift::protocol::T_LIST, 2); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->schema.size())); + std::vector ::const_iterator _iter191; + for (_iter191 = this->schema.begin(); _iter191 != this->schema.end(); ++_iter191) + { + xfer += (*_iter191).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("num_rows", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->num_rows); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("row_groups", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->row_groups.size())); + std::vector ::const_iterator _iter192; + for (_iter192 = this->row_groups.begin(); _iter192 != this->row_groups.end(); ++_iter192) + { + xfer += (*_iter192).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + if (this->__isset.key_value_metadata) { + xfer += oprot->writeFieldBegin("key_value_metadata", ::apache::thrift::protocol::T_LIST, 5); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->key_value_metadata.size())); + std::vector ::const_iterator _iter193; + for (_iter193 = this->key_value_metadata.begin(); _iter193 != this->key_value_metadata.end(); ++_iter193) + { + xfer += (*_iter193).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.created_by) { + xfer += oprot->writeFieldBegin("created_by", ::apache::thrift::protocol::T_STRING, 6); + xfer += oprot->writeString(this->created_by); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.column_orders) { + xfer += oprot->writeFieldBegin("column_orders", ::apache::thrift::protocol::T_LIST, 7); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->column_orders.size())); + std::vector ::const_iterator _iter194; + for (_iter194 = this->column_orders.begin(); _iter194 != this->column_orders.end(); ++_iter194) + { + xfer += (*_iter194).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.encryption_algorithm) { + xfer += oprot->writeFieldBegin("encryption_algorithm", ::apache::thrift::protocol::T_STRUCT, 8); + xfer += this->encryption_algorithm.write(oprot); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.footer_signing_key_metadata) { + xfer += oprot->writeFieldBegin("footer_signing_key_metadata", ::apache::thrift::protocol::T_STRING, 9); + xfer += oprot->writeBinary(this->footer_signing_key_metadata); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(FileMetaData &a, FileMetaData &b) { + using ::std::swap; + swap(a.version, b.version); + swap(a.schema, b.schema); + swap(a.num_rows, b.num_rows); + swap(a.row_groups, b.row_groups); + swap(a.key_value_metadata, b.key_value_metadata); + swap(a.created_by, b.created_by); + swap(a.column_orders, b.column_orders); + swap(a.encryption_algorithm, b.encryption_algorithm); + swap(a.footer_signing_key_metadata, b.footer_signing_key_metadata); + swap(a.__isset, b.__isset); +} + +FileMetaData::FileMetaData(const FileMetaData& other195) { + version = other195.version; + schema = other195.schema; + num_rows = other195.num_rows; + row_groups = other195.row_groups; + key_value_metadata = other195.key_value_metadata; + created_by = other195.created_by; + column_orders = other195.column_orders; + encryption_algorithm = other195.encryption_algorithm; + footer_signing_key_metadata = other195.footer_signing_key_metadata; + __isset = other195.__isset; +} +FileMetaData& FileMetaData::operator=(const FileMetaData& other196) { + version = other196.version; + schema = other196.schema; + num_rows = other196.num_rows; + row_groups = other196.row_groups; + key_value_metadata = other196.key_value_metadata; + created_by = other196.created_by; + column_orders = other196.column_orders; + encryption_algorithm = other196.encryption_algorithm; + footer_signing_key_metadata = other196.footer_signing_key_metadata; + __isset = other196.__isset; + return *this; +} +void FileMetaData::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "FileMetaData("; + out << "version=" << to_string(version); + out << ", " << "schema=" << to_string(schema); + out << ", " << "num_rows=" << to_string(num_rows); + out << ", " << "row_groups=" << to_string(row_groups); + out << ", " << "key_value_metadata="; (__isset.key_value_metadata ? (out << to_string(key_value_metadata)) : (out << "")); + out << ", " << "created_by="; (__isset.created_by ? (out << to_string(created_by)) : (out << "")); + out << ", " << "column_orders="; (__isset.column_orders ? (out << to_string(column_orders)) : (out << "")); + out << ", " << "encryption_algorithm="; (__isset.encryption_algorithm ? (out << to_string(encryption_algorithm)) : (out << "")); + out << ", " << "footer_signing_key_metadata="; (__isset.footer_signing_key_metadata ? (out << to_string(footer_signing_key_metadata)) : (out << "")); + out << ")"; +} + + FileCryptoMetaData::~FileCryptoMetaData() throw() { } @@ -6344,17 +6566,9 @@ void FileCryptoMetaData::__set_encryption_algorithm(const EncryptionAlgorithm& v this->encryption_algorithm = val; } -void FileCryptoMetaData::__set_encrypted_footer(const bool val) { - this->encrypted_footer = val; -} - -void FileCryptoMetaData::__set_footer_key_metadata(const std::string& val) { - this->footer_key_metadata = val; -__isset.footer_key_metadata = true; -} - -void FileCryptoMetaData::__set_footer_offset(const int64_t val) { - this->footer_offset = val; +void FileCryptoMetaData::__set_key_metadata(const std::string& val) { + this->key_metadata = val; +__isset.key_metadata = true; } std::ostream& operator<<(std::ostream& out, const FileCryptoMetaData& obj) { @@ -6376,8 +6590,6 @@ uint32_t FileCryptoMetaData::read(::apache::thrift::protocol::TProtocol* iprot) using ::apache::thrift::protocol::TProtocolException; bool isset_encryption_algorithm = false; - bool isset_encrypted_footer = false; - bool isset_footer_offset = false; while (true) { @@ -6396,25 +6608,9 @@ uint32_t FileCryptoMetaData::read(::apache::thrift::protocol::TProtocol* iprot) } break; case 2: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->encrypted_footer); - isset_encrypted_footer = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 3: if (ftype == ::apache::thrift::protocol::T_STRING) { - xfer += iprot->readBinary(this->footer_key_metadata); - this->__isset.footer_key_metadata = true; - } else { - xfer += iprot->skip(ftype); - } - break; - case 4: - if (ftype == ::apache::thrift::protocol::T_I64) { - xfer += iprot->readI64(this->footer_offset); - isset_footer_offset = true; + xfer += iprot->readBinary(this->key_metadata); + this->__isset.key_metadata = true; } else { xfer += iprot->skip(ftype); } @@ -6430,10 +6626,6 @@ uint32_t FileCryptoMetaData::read(::apache::thrift::protocol::TProtocol* iprot) if (!isset_encryption_algorithm) throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_encrypted_footer) - throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_footer_offset) - throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } @@ -6446,19 +6638,11 @@ uint32_t FileCryptoMetaData::write(::apache::thrift::protocol::TProtocol* oprot) xfer += this->encryption_algorithm.write(oprot); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("encrypted_footer", ::apache::thrift::protocol::T_BOOL, 2); - xfer += oprot->writeBool(this->encrypted_footer); - xfer += oprot->writeFieldEnd(); - - if (this->__isset.footer_key_metadata) { - xfer += oprot->writeFieldBegin("footer_key_metadata", ::apache::thrift::protocol::T_STRING, 3); - xfer += oprot->writeBinary(this->footer_key_metadata); + if (this->__isset.key_metadata) { + xfer += oprot->writeFieldBegin("key_metadata", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->key_metadata); xfer += oprot->writeFieldEnd(); } - xfer += oprot->writeFieldBegin("footer_offset", ::apache::thrift::protocol::T_I64, 4); - xfer += oprot->writeI64(this->footer_offset); - xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -6467,34 +6651,26 @@ uint32_t FileCryptoMetaData::write(::apache::thrift::protocol::TProtocol* oprot) void swap(FileCryptoMetaData &a, FileCryptoMetaData &b) { using ::std::swap; swap(a.encryption_algorithm, b.encryption_algorithm); - swap(a.encrypted_footer, b.encrypted_footer); - swap(a.footer_key_metadata, b.footer_key_metadata); - swap(a.footer_offset, b.footer_offset); + swap(a.key_metadata, b.key_metadata); swap(a.__isset, b.__isset); } -FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other195) { - encryption_algorithm = other195.encryption_algorithm; - encrypted_footer = other195.encrypted_footer; - footer_key_metadata = other195.footer_key_metadata; - footer_offset = other195.footer_offset; - __isset = other195.__isset; +FileCryptoMetaData::FileCryptoMetaData(const FileCryptoMetaData& other197) { + encryption_algorithm = other197.encryption_algorithm; + key_metadata = other197.key_metadata; + __isset = other197.__isset; } -FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other196) { - encryption_algorithm = other196.encryption_algorithm; - encrypted_footer = other196.encrypted_footer; - footer_key_metadata = other196.footer_key_metadata; - footer_offset = other196.footer_offset; - __isset = other196.__isset; +FileCryptoMetaData& FileCryptoMetaData::operator=(const FileCryptoMetaData& other198) { + encryption_algorithm = other198.encryption_algorithm; + key_metadata = other198.key_metadata; + __isset = other198.__isset; return *this; } void FileCryptoMetaData::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "FileCryptoMetaData("; out << "encryption_algorithm=" << to_string(encryption_algorithm); - out << ", " << "encrypted_footer=" << to_string(encrypted_footer); - out << ", " << "footer_key_metadata="; (__isset.footer_key_metadata ? (out << to_string(footer_key_metadata)) : (out << "")); - out << ", " << "footer_offset=" << to_string(footer_offset); + out << ", " << "key_metadata="; (__isset.key_metadata ? (out << to_string(key_metadata)) : (out << "")); out << ")"; } diff --git a/contrib/arrow-cmake/cpp/src/parquet/parquet_types.h b/contrib/arrow-cmake/cpp/src/parquet/parquet_types.h index a10e49be826..65e9e8226bb 100644 --- a/contrib/arrow-cmake/cpp/src/parquet/parquet_types.h +++ b/contrib/arrow-cmake/cpp/src/parquet/parquet_types.h @@ -1,5 +1,5 @@ /** - * Autogenerated by Thrift Compiler (0.11.0) + * Autogenerated by Thrift Compiler (0.12.0) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING * @generated @@ -17,7 +17,7 @@ #include -#include "parquet/util/windows_compatibility.h" +#include "parquet/windows_compatibility.h" namespace parquet { namespace format { @@ -161,6 +161,8 @@ class MilliSeconds; class MicroSeconds; +class NanoSeconds; + class TimeUnit; class TimestampType; @@ -215,14 +217,14 @@ class OffsetIndex; class ColumnIndex; -class FileMetaData; - class AesGcmV1; class AesGcmCtrV1; class EncryptionAlgorithm; +class FileMetaData; + class FileCryptoMetaData; typedef struct _Statistics__isset { @@ -629,10 +631,42 @@ void swap(MicroSeconds &a, MicroSeconds &b); std::ostream& operator<<(std::ostream& out, const MicroSeconds& obj); + +class NanoSeconds : public virtual ::apache::thrift::TBase { + public: + + NanoSeconds(const NanoSeconds&); + NanoSeconds& operator=(const NanoSeconds&); + NanoSeconds() { + } + + virtual ~NanoSeconds() throw(); + + bool operator == (const NanoSeconds & /* rhs */) const + { + return true; + } + bool operator != (const NanoSeconds &rhs) const { + return !(*this == rhs); + } + + bool operator < (const NanoSeconds & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(NanoSeconds &a, NanoSeconds &b); + +std::ostream& operator<<(std::ostream& out, const NanoSeconds& obj); + typedef struct _TimeUnit__isset { - _TimeUnit__isset() : MILLIS(false), MICROS(false) {} + _TimeUnit__isset() : MILLIS(false), MICROS(false), NANOS(false) {} bool MILLIS :1; bool MICROS :1; + bool NANOS :1; } _TimeUnit__isset; class TimeUnit : public virtual ::apache::thrift::TBase { @@ -646,6 +680,7 @@ class TimeUnit : public virtual ::apache::thrift::TBase { virtual ~TimeUnit() throw(); MilliSeconds MILLIS; MicroSeconds MICROS; + NanoSeconds NANOS; _TimeUnit__isset __isset; @@ -653,6 +688,8 @@ class TimeUnit : public virtual ::apache::thrift::TBase { void __set_MICROS(const MicroSeconds& val); + void __set_NANOS(const NanoSeconds& val); + bool operator == (const TimeUnit & rhs) const { if (__isset.MILLIS != rhs.__isset.MILLIS) @@ -663,6 +700,10 @@ class TimeUnit : public virtual ::apache::thrift::TBase { return false; else if (__isset.MICROS && !(MICROS == rhs.MICROS)) return false; + if (__isset.NANOS != rhs.__isset.NANOS) + return false; + else if (__isset.NANOS && !(NANOS == rhs.NANOS)) + return false; return true; } bool operator != (const TimeUnit &rhs) const { @@ -867,7 +908,7 @@ void swap(BsonType &a, BsonType &b); std::ostream& operator<<(std::ostream& out, const BsonType& obj); typedef struct _LogicalType__isset { - _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false) {} + _LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false) {} bool STRING :1; bool MAP :1; bool LIST :1; @@ -880,6 +921,7 @@ typedef struct _LogicalType__isset { bool UNKNOWN :1; bool JSON :1; bool BSON :1; + bool UUID :1; } _LogicalType__isset; class LogicalType : public virtual ::apache::thrift::TBase { @@ -903,6 +945,7 @@ class LogicalType : public virtual ::apache::thrift::TBase { NullType UNKNOWN; JsonType JSON; BsonType BSON; + UUIDType UUID; _LogicalType__isset __isset; @@ -930,6 +973,8 @@ class LogicalType : public virtual ::apache::thrift::TBase { void __set_BSON(const BsonType& val); + void __set_UUID(const UUIDType& val); + bool operator == (const LogicalType & rhs) const { if (__isset.STRING != rhs.__isset.STRING) @@ -980,6 +1025,10 @@ class LogicalType : public virtual ::apache::thrift::TBase { return false; else if (__isset.BSON && !(BSON == rhs.BSON)) return false; + if (__isset.UUID != rhs.__isset.UUID) + return false; + else if (__isset.UUID && !(UUID == rhs.UUID)) + return false; return true; } bool operator != (const LogicalType &rhs) const { @@ -1722,8 +1771,8 @@ void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b); std::ostream& operator<<(std::ostream& out, const EncryptionWithFooterKey& obj); typedef struct _EncryptionWithColumnKey__isset { - _EncryptionWithColumnKey__isset() : column_key_metadata(false) {} - bool column_key_metadata :1; + _EncryptionWithColumnKey__isset() : key_metadata(false) {} + bool key_metadata :1; } _EncryptionWithColumnKey__isset; class EncryptionWithColumnKey : public virtual ::apache::thrift::TBase { @@ -1731,26 +1780,26 @@ class EncryptionWithColumnKey : public virtual ::apache::thrift::TBase { EncryptionWithColumnKey(const EncryptionWithColumnKey&); EncryptionWithColumnKey& operator=(const EncryptionWithColumnKey&); - EncryptionWithColumnKey() : column_key_metadata() { + EncryptionWithColumnKey() : key_metadata() { } virtual ~EncryptionWithColumnKey() throw(); std::vector path_in_schema; - std::string column_key_metadata; + std::string key_metadata; _EncryptionWithColumnKey__isset __isset; void __set_path_in_schema(const std::vector & val); - void __set_column_key_metadata(const std::string& val); + void __set_key_metadata(const std::string& val); bool operator == (const EncryptionWithColumnKey & rhs) const { if (!(path_in_schema == rhs.path_in_schema)) return false; - if (__isset.column_key_metadata != rhs.__isset.column_key_metadata) + if (__isset.key_metadata != rhs.__isset.key_metadata) return false; - else if (__isset.column_key_metadata && !(column_key_metadata == rhs.column_key_metadata)) + else if (__isset.key_metadata && !(key_metadata == rhs.key_metadata)) return false; return true; } @@ -1823,14 +1872,15 @@ void swap(ColumnCryptoMetaData &a, ColumnCryptoMetaData &b); std::ostream& operator<<(std::ostream& out, const ColumnCryptoMetaData& obj); typedef struct _ColumnChunk__isset { - _ColumnChunk__isset() : file_path(false), meta_data(false), offset_index_offset(false), offset_index_length(false), column_index_offset(false), column_index_length(false), crypto_meta_data(false) {} + _ColumnChunk__isset() : file_path(false), meta_data(false), offset_index_offset(false), offset_index_length(false), column_index_offset(false), column_index_length(false), crypto_metadata(false), encrypted_column_metadata(false) {} bool file_path :1; bool meta_data :1; bool offset_index_offset :1; bool offset_index_length :1; bool column_index_offset :1; bool column_index_length :1; - bool crypto_meta_data :1; + bool crypto_metadata :1; + bool encrypted_column_metadata :1; } _ColumnChunk__isset; class ColumnChunk : public virtual ::apache::thrift::TBase { @@ -1838,7 +1888,7 @@ class ColumnChunk : public virtual ::apache::thrift::TBase { ColumnChunk(const ColumnChunk&); ColumnChunk& operator=(const ColumnChunk&); - ColumnChunk() : file_path(), file_offset(0), offset_index_offset(0), offset_index_length(0), column_index_offset(0), column_index_length(0) { + ColumnChunk() : file_path(), file_offset(0), offset_index_offset(0), offset_index_length(0), column_index_offset(0), column_index_length(0), encrypted_column_metadata() { } virtual ~ColumnChunk() throw(); @@ -1849,7 +1899,8 @@ class ColumnChunk : public virtual ::apache::thrift::TBase { int32_t offset_index_length; int64_t column_index_offset; int32_t column_index_length; - ColumnCryptoMetaData crypto_meta_data; + ColumnCryptoMetaData crypto_metadata; + std::string encrypted_column_metadata; _ColumnChunk__isset __isset; @@ -1867,7 +1918,9 @@ class ColumnChunk : public virtual ::apache::thrift::TBase { void __set_column_index_length(const int32_t val); - void __set_crypto_meta_data(const ColumnCryptoMetaData& val); + void __set_crypto_metadata(const ColumnCryptoMetaData& val); + + void __set_encrypted_column_metadata(const std::string& val); bool operator == (const ColumnChunk & rhs) const { @@ -1897,9 +1950,13 @@ class ColumnChunk : public virtual ::apache::thrift::TBase { return false; else if (__isset.column_index_length && !(column_index_length == rhs.column_index_length)) return false; - if (__isset.crypto_meta_data != rhs.__isset.crypto_meta_data) + if (__isset.crypto_metadata != rhs.__isset.crypto_metadata) return false; - else if (__isset.crypto_meta_data && !(crypto_meta_data == rhs.crypto_meta_data)) + else if (__isset.crypto_metadata && !(crypto_metadata == rhs.crypto_metadata)) + return false; + if (__isset.encrypted_column_metadata != rhs.__isset.encrypted_column_metadata) + return false; + else if (__isset.encrypted_column_metadata && !(encrypted_column_metadata == rhs.encrypted_column_metadata)) return false; return true; } @@ -1920,10 +1977,11 @@ void swap(ColumnChunk &a, ColumnChunk &b); std::ostream& operator<<(std::ostream& out, const ColumnChunk& obj); typedef struct _RowGroup__isset { - _RowGroup__isset() : sorting_columns(false), file_offset(false), total_compressed_size(false) {} + _RowGroup__isset() : sorting_columns(false), file_offset(false), total_compressed_size(false), ordinal(false) {} bool sorting_columns :1; bool file_offset :1; bool total_compressed_size :1; + bool ordinal :1; } _RowGroup__isset; class RowGroup : public virtual ::apache::thrift::TBase { @@ -1931,7 +1989,7 @@ class RowGroup : public virtual ::apache::thrift::TBase { RowGroup(const RowGroup&); RowGroup& operator=(const RowGroup&); - RowGroup() : total_byte_size(0), num_rows(0), file_offset(0), total_compressed_size(0) { + RowGroup() : total_byte_size(0), num_rows(0), file_offset(0), total_compressed_size(0), ordinal(0) { } virtual ~RowGroup() throw(); @@ -1941,6 +1999,7 @@ class RowGroup : public virtual ::apache::thrift::TBase { std::vector sorting_columns; int64_t file_offset; int64_t total_compressed_size; + int16_t ordinal; _RowGroup__isset __isset; @@ -1956,6 +2015,8 @@ class RowGroup : public virtual ::apache::thrift::TBase { void __set_total_compressed_size(const int64_t val); + void __set_ordinal(const int16_t val); + bool operator == (const RowGroup & rhs) const { if (!(columns == rhs.columns)) @@ -1976,6 +2037,10 @@ class RowGroup : public virtual ::apache::thrift::TBase { return false; else if (__isset.total_compressed_size && !(total_compressed_size == rhs.total_compressed_size)) return false; + if (__isset.ordinal != rhs.__isset.ordinal) + return false; + else if (__isset.ordinal && !(ordinal == rhs.ordinal)) + return false; return true; } bool operator != (const RowGroup &rhs) const { @@ -2215,90 +2280,11 @@ void swap(ColumnIndex &a, ColumnIndex &b); std::ostream& operator<<(std::ostream& out, const ColumnIndex& obj); -typedef struct _FileMetaData__isset { - _FileMetaData__isset() : key_value_metadata(false), created_by(false), column_orders(false) {} - bool key_value_metadata :1; - bool created_by :1; - bool column_orders :1; -} _FileMetaData__isset; - -class FileMetaData : public virtual ::apache::thrift::TBase { - public: - - FileMetaData(const FileMetaData&); - FileMetaData& operator=(const FileMetaData&); - FileMetaData() : version(0), num_rows(0), created_by() { - } - - virtual ~FileMetaData() throw(); - int32_t version; - std::vector schema; - int64_t num_rows; - std::vector row_groups; - std::vector key_value_metadata; - std::string created_by; - std::vector column_orders; - - _FileMetaData__isset __isset; - - void __set_version(const int32_t val); - - void __set_schema(const std::vector & val); - - void __set_num_rows(const int64_t val); - - void __set_row_groups(const std::vector & val); - - void __set_key_value_metadata(const std::vector & val); - - void __set_created_by(const std::string& val); - - void __set_column_orders(const std::vector & val); - - bool operator == (const FileMetaData & rhs) const - { - if (!(version == rhs.version)) - return false; - if (!(schema == rhs.schema)) - return false; - if (!(num_rows == rhs.num_rows)) - return false; - if (!(row_groups == rhs.row_groups)) - return false; - if (__isset.key_value_metadata != rhs.__isset.key_value_metadata) - return false; - else if (__isset.key_value_metadata && !(key_value_metadata == rhs.key_value_metadata)) - return false; - if (__isset.created_by != rhs.__isset.created_by) - return false; - else if (__isset.created_by && !(created_by == rhs.created_by)) - return false; - if (__isset.column_orders != rhs.__isset.column_orders) - return false; - else if (__isset.column_orders && !(column_orders == rhs.column_orders)) - return false; - return true; - } - bool operator != (const FileMetaData &rhs) const { - return !(*this == rhs); - } - - bool operator < (const FileMetaData & ) const; - - uint32_t read(::apache::thrift::protocol::TProtocol* iprot); - uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; - - virtual void printTo(std::ostream& out) const; -}; - -void swap(FileMetaData &a, FileMetaData &b); - -std::ostream& operator<<(std::ostream& out, const FileMetaData& obj); - typedef struct _AesGcmV1__isset { - _AesGcmV1__isset() : aad_metadata(false), iv_prefix(false) {} - bool aad_metadata :1; - bool iv_prefix :1; + _AesGcmV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} + bool aad_prefix :1; + bool aad_file_unique :1; + bool supply_aad_prefix :1; } _AesGcmV1__isset; class AesGcmV1 : public virtual ::apache::thrift::TBase { @@ -2306,28 +2292,35 @@ class AesGcmV1 : public virtual ::apache::thrift::TBase { AesGcmV1(const AesGcmV1&); AesGcmV1& operator=(const AesGcmV1&); - AesGcmV1() : aad_metadata(), iv_prefix() { + AesGcmV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) { } virtual ~AesGcmV1() throw(); - std::string aad_metadata; - std::string iv_prefix; + std::string aad_prefix; + std::string aad_file_unique; + bool supply_aad_prefix; _AesGcmV1__isset __isset; - void __set_aad_metadata(const std::string& val); + void __set_aad_prefix(const std::string& val); - void __set_iv_prefix(const std::string& val); + void __set_aad_file_unique(const std::string& val); + + void __set_supply_aad_prefix(const bool val); bool operator == (const AesGcmV1 & rhs) const { - if (__isset.aad_metadata != rhs.__isset.aad_metadata) + if (__isset.aad_prefix != rhs.__isset.aad_prefix) return false; - else if (__isset.aad_metadata && !(aad_metadata == rhs.aad_metadata)) + else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) return false; - if (__isset.iv_prefix != rhs.__isset.iv_prefix) + if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) return false; - else if (__isset.iv_prefix && !(iv_prefix == rhs.iv_prefix)) + else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) + return false; + if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) + return false; + else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) return false; return true; } @@ -2348,10 +2341,10 @@ void swap(AesGcmV1 &a, AesGcmV1 &b); std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj); typedef struct _AesGcmCtrV1__isset { - _AesGcmCtrV1__isset() : aad_metadata(false), gcm_iv_prefix(false), ctr_iv_prefix(false) {} - bool aad_metadata :1; - bool gcm_iv_prefix :1; - bool ctr_iv_prefix :1; + _AesGcmCtrV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {} + bool aad_prefix :1; + bool aad_file_unique :1; + bool supply_aad_prefix :1; } _AesGcmCtrV1__isset; class AesGcmCtrV1 : public virtual ::apache::thrift::TBase { @@ -2359,35 +2352,35 @@ class AesGcmCtrV1 : public virtual ::apache::thrift::TBase { AesGcmCtrV1(const AesGcmCtrV1&); AesGcmCtrV1& operator=(const AesGcmCtrV1&); - AesGcmCtrV1() : aad_metadata(), gcm_iv_prefix(), ctr_iv_prefix() { + AesGcmCtrV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) { } virtual ~AesGcmCtrV1() throw(); - std::string aad_metadata; - std::string gcm_iv_prefix; - std::string ctr_iv_prefix; + std::string aad_prefix; + std::string aad_file_unique; + bool supply_aad_prefix; _AesGcmCtrV1__isset __isset; - void __set_aad_metadata(const std::string& val); + void __set_aad_prefix(const std::string& val); - void __set_gcm_iv_prefix(const std::string& val); + void __set_aad_file_unique(const std::string& val); - void __set_ctr_iv_prefix(const std::string& val); + void __set_supply_aad_prefix(const bool val); bool operator == (const AesGcmCtrV1 & rhs) const { - if (__isset.aad_metadata != rhs.__isset.aad_metadata) + if (__isset.aad_prefix != rhs.__isset.aad_prefix) return false; - else if (__isset.aad_metadata && !(aad_metadata == rhs.aad_metadata)) + else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix)) return false; - if (__isset.gcm_iv_prefix != rhs.__isset.gcm_iv_prefix) + if (__isset.aad_file_unique != rhs.__isset.aad_file_unique) return false; - else if (__isset.gcm_iv_prefix && !(gcm_iv_prefix == rhs.gcm_iv_prefix)) + else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique)) return false; - if (__isset.ctr_iv_prefix != rhs.__isset.ctr_iv_prefix) + if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix) return false; - else if (__isset.ctr_iv_prefix && !(ctr_iv_prefix == rhs.ctr_iv_prefix)) + else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix)) return false; return true; } @@ -2459,9 +2452,105 @@ void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b); std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj); +typedef struct _FileMetaData__isset { + _FileMetaData__isset() : key_value_metadata(false), created_by(false), column_orders(false), encryption_algorithm(false), footer_signing_key_metadata(false) {} + bool key_value_metadata :1; + bool created_by :1; + bool column_orders :1; + bool encryption_algorithm :1; + bool footer_signing_key_metadata :1; +} _FileMetaData__isset; + +class FileMetaData : public virtual ::apache::thrift::TBase { + public: + + FileMetaData(const FileMetaData&); + FileMetaData& operator=(const FileMetaData&); + FileMetaData() : version(0), num_rows(0), created_by(), footer_signing_key_metadata() { + } + + virtual ~FileMetaData() throw(); + int32_t version; + std::vector schema; + int64_t num_rows; + std::vector row_groups; + std::vector key_value_metadata; + std::string created_by; + std::vector column_orders; + EncryptionAlgorithm encryption_algorithm; + std::string footer_signing_key_metadata; + + _FileMetaData__isset __isset; + + void __set_version(const int32_t val); + + void __set_schema(const std::vector & val); + + void __set_num_rows(const int64_t val); + + void __set_row_groups(const std::vector & val); + + void __set_key_value_metadata(const std::vector & val); + + void __set_created_by(const std::string& val); + + void __set_column_orders(const std::vector & val); + + void __set_encryption_algorithm(const EncryptionAlgorithm& val); + + void __set_footer_signing_key_metadata(const std::string& val); + + bool operator == (const FileMetaData & rhs) const + { + if (!(version == rhs.version)) + return false; + if (!(schema == rhs.schema)) + return false; + if (!(num_rows == rhs.num_rows)) + return false; + if (!(row_groups == rhs.row_groups)) + return false; + if (__isset.key_value_metadata != rhs.__isset.key_value_metadata) + return false; + else if (__isset.key_value_metadata && !(key_value_metadata == rhs.key_value_metadata)) + return false; + if (__isset.created_by != rhs.__isset.created_by) + return false; + else if (__isset.created_by && !(created_by == rhs.created_by)) + return false; + if (__isset.column_orders != rhs.__isset.column_orders) + return false; + else if (__isset.column_orders && !(column_orders == rhs.column_orders)) + return false; + if (__isset.encryption_algorithm != rhs.__isset.encryption_algorithm) + return false; + else if (__isset.encryption_algorithm && !(encryption_algorithm == rhs.encryption_algorithm)) + return false; + if (__isset.footer_signing_key_metadata != rhs.__isset.footer_signing_key_metadata) + return false; + else if (__isset.footer_signing_key_metadata && !(footer_signing_key_metadata == rhs.footer_signing_key_metadata)) + return false; + return true; + } + bool operator != (const FileMetaData &rhs) const { + return !(*this == rhs); + } + + bool operator < (const FileMetaData & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(FileMetaData &a, FileMetaData &b); + +std::ostream& operator<<(std::ostream& out, const FileMetaData& obj); + typedef struct _FileCryptoMetaData__isset { - _FileCryptoMetaData__isset() : footer_key_metadata(false) {} - bool footer_key_metadata :1; + _FileCryptoMetaData__isset() : key_metadata(false) {} + bool key_metadata :1; } _FileCryptoMetaData__isset; class FileCryptoMetaData : public virtual ::apache::thrift::TBase { @@ -2469,36 +2558,26 @@ class FileCryptoMetaData : public virtual ::apache::thrift::TBase { FileCryptoMetaData(const FileCryptoMetaData&); FileCryptoMetaData& operator=(const FileCryptoMetaData&); - FileCryptoMetaData() : encrypted_footer(0), footer_key_metadata(), footer_offset(0) { + FileCryptoMetaData() : key_metadata() { } virtual ~FileCryptoMetaData() throw(); EncryptionAlgorithm encryption_algorithm; - bool encrypted_footer; - std::string footer_key_metadata; - int64_t footer_offset; + std::string key_metadata; _FileCryptoMetaData__isset __isset; void __set_encryption_algorithm(const EncryptionAlgorithm& val); - void __set_encrypted_footer(const bool val); - - void __set_footer_key_metadata(const std::string& val); - - void __set_footer_offset(const int64_t val); + void __set_key_metadata(const std::string& val); bool operator == (const FileCryptoMetaData & rhs) const { if (!(encryption_algorithm == rhs.encryption_algorithm)) return false; - if (!(encrypted_footer == rhs.encrypted_footer)) + if (__isset.key_metadata != rhs.__isset.key_metadata) return false; - if (__isset.footer_key_metadata != rhs.__isset.footer_key_metadata) - return false; - else if (__isset.footer_key_metadata && !(footer_key_metadata == rhs.footer_key_metadata)) - return false; - if (!(footer_offset == rhs.footer_offset)) + else if (__isset.key_metadata && !(key_metadata == rhs.key_metadata)) return false; return true; } diff --git a/contrib/flatbuffers b/contrib/flatbuffers new file mode 160000 index 00000000000..bf9eb67ab93 --- /dev/null +++ b/contrib/flatbuffers @@ -0,0 +1 @@ +Subproject commit bf9eb67ab9371755c6bcece13cadc7693bcbf264 diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 77465daad27..972401c19d5 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -298,7 +298,7 @@ int Server::main(const std::vector & /*args*/) global_context->shutdown(); - LOG_DEBUG(log, "Shutted down storages."); + LOG_DEBUG(log, "Shut down storages."); /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. diff --git a/dbms/programs/server/users.xml b/dbms/programs/server/users.xml index 9755c29d480..0058ee51184 100644 --- a/dbms/programs/server/users.xml +++ b/dbms/programs/server/users.xml @@ -84,25 +84,17 @@ default - - + + a = 1 - - a + b < 1 or c - d > 5 - - - - c = 1 - - + --> diff --git a/dbms/src/Access/AccessControlManager.cpp b/dbms/src/Access/AccessControlManager.cpp index 1f1a57816a8..249dc54fb09 100644 --- a/dbms/src/Access/AccessControlManager.cpp +++ b/dbms/src/Access/AccessControlManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -21,7 +22,8 @@ namespace AccessControlManager::AccessControlManager() : MultipleAccessStorage(createStorages()), - quota_context_factory(std::make_unique(*this)) + quota_context_factory(std::make_unique(*this)), + row_policy_context_factory(std::make_unique(*this)) { } @@ -49,4 +51,11 @@ std::vector AccessControlManager::getQuotaUsageInfo() const { return quota_context_factory->getUsageInfo(); } + + +std::shared_ptr AccessControlManager::getRowPolicyContext(const String & user_name) const +{ + return row_policy_context_factory->createContext(user_name); +} + } diff --git a/dbms/src/Access/AccessControlManager.h b/dbms/src/Access/AccessControlManager.h index 2133717d676..9658dc7161d 100644 --- a/dbms/src/Access/AccessControlManager.h +++ b/dbms/src/Access/AccessControlManager.h @@ -22,6 +22,8 @@ namespace DB class QuotaContext; class QuotaContextFactory; struct QuotaUsageInfo; +class RowPolicyContext; +class RowPolicyContextFactory; /// Manages access control entities. @@ -38,8 +40,11 @@ public: std::vector getQuotaUsageInfo() const; + std::shared_ptr getRowPolicyContext(const String & user_name) const; + private: std::unique_ptr quota_context_factory; + std::unique_ptr row_policy_context_factory; }; } diff --git a/dbms/src/Access/IAccessEntity.cpp b/dbms/src/Access/IAccessEntity.cpp index 6a2f928ae9e..9b203a74148 100644 --- a/dbms/src/Access/IAccessEntity.cpp +++ b/dbms/src/Access/IAccessEntity.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -9,6 +10,8 @@ String IAccessEntity::getTypeName(std::type_index type) { if (type == typeid(Quota)) return "Quota"; + if (type == typeid(RowPolicy)) + return "Row policy"; return demangle(type.name()); } diff --git a/dbms/src/Access/RowPolicy.cpp b/dbms/src/Access/RowPolicy.cpp new file mode 100644 index 00000000000..391303e46a2 --- /dev/null +++ b/dbms/src/Access/RowPolicy.cpp @@ -0,0 +1,111 @@ +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_) + { + full_name_.clear(); + full_name_.reserve(database_.length() + table_name_.length() + policy_name_.length() + 6); + full_name_ += backQuoteIfNeed(policy_name_); + full_name_ += " ON "; + if (!database_.empty()) + { + full_name_ += backQuoteIfNeed(database_); + full_name_ += '.'; + } + full_name_ += backQuoteIfNeed(table_name_); + } +} + + +String RowPolicy::FullNameParts::getFullName() const +{ + String full_name; + generateFullNameImpl(database, table_name, policy_name, full_name); + return full_name; +} + + +String RowPolicy::FullNameParts::getFullName(const Context & context) const +{ + String full_name; + generateFullNameImpl(database.empty() ? context.getCurrentDatabase() : database, table_name, policy_name, full_name); + return full_name; +} + + +void RowPolicy::setDatabase(const String & database_) +{ + database = database_; + generateFullNameImpl(database, table_name, policy_name, full_name); +} + + +void RowPolicy::setTableName(const String & table_name_) +{ + table_name = table_name_; + generateFullNameImpl(database, table_name, policy_name, full_name); +} + + +void RowPolicy::setName(const String & policy_name_) +{ + policy_name = policy_name_; + generateFullNameImpl(database, table_name, policy_name, full_name); +} + + +void RowPolicy::setFullName(const String & database_, const String & table_name_, const String & policy_name_) +{ + database = database_; + table_name = table_name_; + policy_name = policy_name_; + generateFullNameImpl(database, table_name, policy_name, full_name); +} + + +bool RowPolicy::equal(const IAccessEntity & other) const +{ + if (!IAccessEntity::equal(other)) + return false; + const auto & other_policy = typeid_cast(other); + return (database == other_policy.database) && (table_name == other_policy.table_name) && (policy_name == other_policy.policy_name) + && boost::range::equal(conditions, other_policy.conditions) && restrictive == other_policy.restrictive + && (roles == other_policy.roles) && (all_roles == other_policy.all_roles) && (except_roles == other_policy.except_roles); +} + + +const char * RowPolicy::conditionIndexToString(ConditionIndex index) +{ + switch (index) + { + case SELECT_FILTER: return "SELECT_FILTER"; + case INSERT_CHECK: return "INSERT_CHECK"; + case UPDATE_FILTER: return "UPDATE_FILTER"; + case UPDATE_CHECK: return "UPDATE_CHECK"; + case DELETE_FILTER: return "DELETE_FILTER"; + } + __builtin_unreachable(); +} + + +const char * RowPolicy::conditionIndexToColumnName(ConditionIndex index) +{ + switch (index) + { + case SELECT_FILTER: return "select_filter"; + case INSERT_CHECK: return "insert_check"; + case UPDATE_FILTER: return "update_filter"; + case UPDATE_CHECK: return "update_check"; + case DELETE_FILTER: return "delete_filter"; + } + __builtin_unreachable(); +} + +} diff --git a/dbms/src/Access/RowPolicy.h b/dbms/src/Access/RowPolicy.h new file mode 100644 index 00000000000..22681b8875e --- /dev/null +++ b/dbms/src/Access/RowPolicy.h @@ -0,0 +1,81 @@ +#pragma once + +#include + + +namespace DB +{ +class Context; + + +/** Represents a row level security policy for a table. + */ +struct RowPolicy : public IAccessEntity +{ + void setDatabase(const String & database_); + void setTableName(const String & table_name_); + void setName(const String & policy_name_) override; + void setFullName(const String & database_, const String & table_name_, const String & policy_name_); + + String getDatabase() const { return database; } + String getTableName() const { return table_name; } + String getName() const override { return policy_name; } + + struct FullNameParts + { + String database; + String table_name; + String policy_name; + String getFullName() const; + String getFullName(const Context & context) const; + }; + + /// Filter is a SQL conditional expression used to figure out which rows should be visible + /// for user or available for modification. If the expression returns NULL or false for some rows + /// those rows are silently suppressed. + /// Check is a SQL condition expression used to check whether a row can be written into + /// the table. If the expression returns NULL or false an exception is thrown. + /// If a conditional expression here is empty it means no filtering is applied. + enum ConditionIndex + { + SELECT_FILTER, + INSERT_CHECK, + UPDATE_FILTER, + UPDATE_CHECK, + DELETE_FILTER, + }; + static constexpr size_t MAX_CONDITION_INDEX = 5; + static const char * conditionIndexToString(ConditionIndex index); + static const char * conditionIndexToColumnName(ConditionIndex index); + + String conditions[MAX_CONDITION_INDEX]; + + /// Sets that the policy is permissive. + /// A row is only accessible if at least one of the permissive policies passes, + /// in addition to all the restrictive policies. + void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); } + bool isPermissive() const { return !isRestrictive(); } + + /// Sets that the policy is restrictive. + /// A row is only accessible if at least one of the permissive policies passes, + /// in addition to all the restrictive policies. + void setRestrictive(bool restrictive_ = true) { restrictive = restrictive_; } + bool isRestrictive() const { return restrictive; } + + bool equal(const IAccessEntity & other) const override; + std::shared_ptr clone() const override { return cloneImpl(); } + + /// Which roles or users should use this quota. + Strings roles; + bool all_roles = false; + Strings except_roles; + +private: + String database; + String table_name; + String policy_name; + bool restrictive = false; +}; + +using RowPolicyPtr = std::shared_ptr; +} diff --git a/dbms/src/Access/RowPolicyContext.cpp b/dbms/src/Access/RowPolicyContext.cpp new file mode 100644 index 00000000000..cb24d0af01b --- /dev/null +++ b/dbms/src/Access/RowPolicyContext.cpp @@ -0,0 +1,59 @@ +#include +#include +#include + + +namespace DB +{ +size_t RowPolicyContext::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const +{ + return std::hash{}(database_and_table_name.first) - std::hash{}(database_and_table_name.second); +} + + +RowPolicyContext::RowPolicyContext() + : atomic_map_of_mixed_conditions(std::make_shared()) +{ +} + + +RowPolicyContext::~RowPolicyContext() = default; + + +RowPolicyContext::RowPolicyContext(const String & user_name_) + : user_name(user_name_) +{} + + +ASTPtr RowPolicyContext::getCondition(const String & database, const String & table_name, ConditionIndex index) const +{ + /// We don't lock `mutex` here. + auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions); + auto it = map_of_mixed_conditions->find({database, table_name}); + if (it == map_of_mixed_conditions->end()) + return {}; + return it->second.mixed_conditions[index]; +} + + +std::vector RowPolicyContext::getCurrentPolicyIDs() const +{ + /// We don't lock `mutex` here. + auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions); + std::vector policy_ids; + for (const auto & mixed_conditions : *map_of_mixed_conditions | boost::adaptors::map_values) + boost::range::copy(mixed_conditions.policy_ids, std::back_inserter(policy_ids)); + return policy_ids; +} + + +std::vector RowPolicyContext::getCurrentPolicyIDs(const String & database, const String & table_name) const +{ + /// We don't lock `mutex` here. + auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions); + auto it = map_of_mixed_conditions->find({database, table_name}); + if (it == map_of_mixed_conditions->end()) + return {}; + return it->second.policy_ids; +} +} diff --git a/dbms/src/Access/RowPolicyContext.h b/dbms/src/Access/RowPolicyContext.h new file mode 100644 index 00000000000..776808f74d7 --- /dev/null +++ b/dbms/src/Access/RowPolicyContext.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +class IAST; +using ASTPtr = std::shared_ptr; + + +/// Provides fast access to row policies' conditions for a specific user and tables. +class RowPolicyContext +{ +public: + /// Default constructor makes a row policy usage context which restricts nothing. + RowPolicyContext(); + + ~RowPolicyContext(); + + using ConditionIndex = RowPolicy::ConditionIndex; + + /// Returns prepared filter for a specific table and operations. + /// The function can return nullptr, that means there is no filters applied. + /// The returned filter can be a combination of the filters defined by multiple row policies. + ASTPtr getCondition(const String & database, const String & table_name, ConditionIndex index) const; + + /// Returns IDs of all the policies used by the current user. + std::vector getCurrentPolicyIDs() const; + + /// Returns IDs of the policies used by a concrete table. + std::vector getCurrentPolicyIDs(const String & database, const String & table_name) const; + +private: + friend class RowPolicyContextFactory; + friend struct ext::shared_ptr_helper; + RowPolicyContext(const String & user_name_); /// RowPolicyContext should be created by RowPolicyContextFactory. + + using DatabaseAndTableName = std::pair; + using DatabaseAndTableNameRef = std::pair; + struct Hash + { + size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const; + }; + static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX; + using ParsedConditions = std::array; + struct MixedConditions + { + std::unique_ptr database_and_table_name_keeper; + ParsedConditions mixed_conditions; + std::vector policy_ids; + }; + using MapOfMixedConditions = std::unordered_map; + + const String user_name; + std::shared_ptr atomic_map_of_mixed_conditions; /// Changed atomically, not protected by `mutex`. +}; + + +using RowPolicyContextPtr = std::shared_ptr; +} diff --git a/dbms/src/Access/RowPolicyContextFactory.cpp b/dbms/src/Access/RowPolicyContextFactory.cpp new file mode 100644 index 00000000000..e458f06ca94 --- /dev/null +++ b/dbms/src/Access/RowPolicyContextFactory.cpp @@ -0,0 +1,314 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + bool tryGetLiteralBool(const IAST & ast, bool & value) + { + try + { + if (const ASTLiteral * literal = ast.as()) + { + value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber(), literal->value); + return true; + } + return false; + } + catch (...) + { + return false; + } + } + + ASTPtr applyFunctionAND(ASTs arguments) + { + bool const_arguments = true; + boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool + { + bool b; + if (!tryGetLiteralBool(*argument, b)) + return false; + const_arguments &= b; + return true; + }); + + if (!const_arguments) + return std::make_shared(Field{UInt8(0)}); + if (arguments.empty()) + return std::make_shared(Field{UInt8(1)}); + if (arguments.size() == 1) + return arguments[0]; + + auto function = std::make_shared(); + auto exp_list = std::make_shared(); + function->name = "and"; + function->arguments = exp_list; + function->children.push_back(exp_list); + exp_list->children = std::move(arguments); + return function; + } + + + ASTPtr applyFunctionOR(ASTs arguments) + { + bool const_arguments = false; + boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool + { + bool b; + if (!tryGetLiteralBool(*argument, b)) + return false; + const_arguments |= b; + return true; + }); + + if (const_arguments) + return std::make_shared(Field{UInt8(1)}); + if (arguments.empty()) + return std::make_shared(Field{UInt8(0)}); + if (arguments.size() == 1) + return arguments[0]; + + auto function = std::make_shared(); + auto exp_list = std::make_shared(); + function->name = "or"; + function->arguments = exp_list; + function->children.push_back(exp_list); + exp_list->children = std::move(arguments); + return function; + } + + + using ConditionIndex = RowPolicy::ConditionIndex; + static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX; + + + /// Accumulates conditions from multiple row policies and joins them using the AND logical operation. + class ConditionsMixer + { + public: + void add(const ASTPtr & condition, bool is_restrictive) + { + if (!condition) + return; + + if (is_restrictive) + restrictions.push_back(condition); + else + permissions.push_back(condition); + } + + ASTPtr getResult() && + { + /// Process permissive conditions. + if (!permissions.empty()) + restrictions.push_back(applyFunctionOR(std::move(permissions))); + + /// Process restrictive conditions. + if (!restrictions.empty()) + return applyFunctionAND(std::move(restrictions)); + return nullptr; + } + + private: + ASTs permissions; + ASTs restrictions; + }; +} + + +void RowPolicyContextFactory::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) +{ + policy = policy_; + + boost::range::copy(policy->roles, std::inserter(roles, roles.end())); + all_roles = policy->all_roles; + boost::range::copy(policy->except_roles, std::inserter(except_roles, except_roles.end())); + + for (auto index : ext::range_with_static_cast(0, MAX_CONDITION_INDEX)) + { + const String & condition = policy->conditions[index]; + auto previous_range = std::pair(std::begin(policy->conditions), std::begin(policy->conditions) + index); + auto previous_it = std::find(previous_range.first, previous_range.second, condition); + if (previous_it != previous_range.second) + { + /// The condition is already parsed before. + parsed_conditions[index] = parsed_conditions[previous_it - previous_range.first]; + } + else + { + /// Try to parse the condition. + try + { + ParserExpression parser; + parsed_conditions[index] = parseQuery(parser, condition, 0); + } + catch (...) + { + tryLogCurrentException( + &Poco::Logger::get("RowPolicy"), + String("Could not parse the condition ") + RowPolicy::conditionIndexToString(index) + " of row policy " + + backQuote(policy->getFullName())); + } + } + } +} + + +bool RowPolicyContextFactory::PolicyInfo::canUseWithContext(const RowPolicyContext & context) const +{ + if (roles.count(context.user_name)) + return true; + + if (all_roles && !except_roles.count(context.user_name)) + return true; + + return false; +} + + +RowPolicyContextFactory::RowPolicyContextFactory(const AccessControlManager & access_control_manager_) + : access_control_manager(access_control_manager_) +{ +} + +RowPolicyContextFactory::~RowPolicyContextFactory() = default; + + +RowPolicyContextPtr RowPolicyContextFactory::createContext(const String & user_name) +{ + std::lock_guard lock{mutex}; + ensureAllRowPoliciesRead(); + auto context = ext::shared_ptr_helper::create(user_name); + contexts.push_back(context); + mixConditionsForContext(*context); + return context; +} + + +void RowPolicyContextFactory::ensureAllRowPoliciesRead() +{ + /// `mutex` is already locked. + if (all_policies_read) + return; + all_policies_read = true; + + subscription = access_control_manager.subscribeForChanges( + [&](const UUID & id, const AccessEntityPtr & entity) + { + if (entity) + rowPolicyAddedOrChanged(id, typeid_cast(entity)); + else + rowPolicyRemoved(id); + }); + + for (const UUID & id : access_control_manager.findAll()) + { + auto quota = access_control_manager.tryRead(id); + if (quota) + all_policies.emplace(id, PolicyInfo(quota)); + } +} + + +void RowPolicyContextFactory::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy) +{ + std::lock_guard lock{mutex}; + auto it = all_policies.find(policy_id); + if (it == all_policies.end()) + { + it = all_policies.emplace(policy_id, PolicyInfo(new_policy)).first; + } + else + { + if (it->second.policy == new_policy) + return; + } + + auto & info = it->second; + info.setPolicy(new_policy); + mixConditionsForAllContexts(); +} + + +void RowPolicyContextFactory::rowPolicyRemoved(const UUID & policy_id) +{ + std::lock_guard lock{mutex}; + all_policies.erase(policy_id); + mixConditionsForAllContexts(); +} + + +void RowPolicyContextFactory::mixConditionsForAllContexts() +{ + /// `mutex` is already locked. + boost::range::remove_erase_if( + contexts, + [&](const std::weak_ptr & weak) + { + auto context = weak.lock(); + if (!context) + return true; // remove from the `contexts` list. + mixConditionsForContext(*context); + return false; // keep in the `contexts` list. + }); +} + + +void RowPolicyContextFactory::mixConditionsForContext(RowPolicyContext & context) +{ + /// `mutex` is already locked. + struct Mixers + { + ConditionsMixer mixers[MAX_CONDITION_INDEX]; + std::vector policy_ids; + }; + using MapOfMixedConditions = RowPolicyContext::MapOfMixedConditions; + using DatabaseAndTableName = RowPolicyContext::DatabaseAndTableName; + using DatabaseAndTableNameRef = RowPolicyContext::DatabaseAndTableNameRef; + using Hash = RowPolicyContext::Hash; + + std::unordered_map map_of_mixers; + + for (const auto & [policy_id, info] : all_policies) + { + if (info.canUseWithContext(context)) + { + const auto & policy = *info.policy; + auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}]; + mixers.policy_ids.push_back(policy_id); + for (auto index : ext::range(0, MAX_CONDITION_INDEX)) + mixers.mixers[index].add(info.parsed_conditions[index], policy.isRestrictive()); + } + } + + auto map_of_mixed_conditions = std::make_shared(); + for (auto & [database_and_table_name, mixers] : map_of_mixers) + { + auto database_and_table_name_keeper = std::make_unique(); + database_and_table_name_keeper->first = database_and_table_name.first; + database_and_table_name_keeper->second = database_and_table_name.second; + auto & mixed_conditions = (*map_of_mixed_conditions)[DatabaseAndTableNameRef{database_and_table_name_keeper->first, + database_and_table_name_keeper->second}]; + mixed_conditions.database_and_table_name_keeper = std::move(database_and_table_name_keeper); + mixed_conditions.policy_ids = std::move(mixers.policy_ids); + for (auto index : ext::range(0, MAX_CONDITION_INDEX)) + mixed_conditions.mixed_conditions[index] = std::move(mixers.mixers[index]).getResult(); + } + + std::atomic_store(&context.atomic_map_of_mixed_conditions, std::shared_ptr{map_of_mixed_conditions}); +} + +} diff --git a/dbms/src/Access/RowPolicyContextFactory.h b/dbms/src/Access/RowPolicyContextFactory.h new file mode 100644 index 00000000000..02019c468a9 --- /dev/null +++ b/dbms/src/Access/RowPolicyContextFactory.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ +class AccessControlManager; + + +/// Stores read and parsed row policies. +class RowPolicyContextFactory +{ +public: + RowPolicyContextFactory(const AccessControlManager & access_control_manager_); + ~RowPolicyContextFactory(); + + RowPolicyContextPtr createContext(const String & user_name); + +private: + using ParsedConditions = RowPolicyContext::ParsedConditions; + + struct PolicyInfo + { + PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); } + void setPolicy(const RowPolicyPtr & policy_); + bool canUseWithContext(const RowPolicyContext & context) const; + + RowPolicyPtr policy; + std::unordered_set roles; + bool all_roles = false; + std::unordered_set except_roles; + ParsedConditions parsed_conditions; + }; + + void ensureAllRowPoliciesRead(); + void rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy); + void rowPolicyRemoved(const UUID & policy_id); + void mixConditionsForAllContexts(); + void mixConditionsForContext(RowPolicyContext & context); + + const AccessControlManager & access_control_manager; + std::unordered_map all_policies; + bool all_policies_read = false; + IAccessStorage::SubscriptionPtr subscription; + std::vector> contexts; + std::mutex mutex; +}; + +} diff --git a/dbms/src/Access/UsersConfigAccessStorage.cpp b/dbms/src/Access/UsersConfigAccessStorage.cpp index d417968bb64..c9671afaca1 100644 --- a/dbms/src/Access/UsersConfigAccessStorage.cpp +++ b/dbms/src/Access/UsersConfigAccessStorage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ namespace { if (type == typeid(Quota)) return 'Q'; + if (type == typeid(RowPolicy)) + return 'P'; return 0; } @@ -112,6 +115,57 @@ namespace } return quotas; } + + + std::vector parseRowPolicies(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) + { + std::vector policies; + Poco::Util::AbstractConfiguration::Keys user_names; + config.keys("users", user_names); + + for (const String & user_name : user_names) + { + const String databases_config = "users." + user_name + ".databases"; + if (config.has(databases_config)) + { + Poco::Util::AbstractConfiguration::Keys databases; + config.keys(databases_config, databases); + + /// Read tables within databases + for (const String & database : databases) + { + const String database_config = databases_config + "." + database; + Poco::Util::AbstractConfiguration::Keys table_names; + config.keys(database_config, table_names); + + /// Read table properties + for (const String & table_name : table_names) + { + const auto filter_config = database_config + "." + table_name + ".filter"; + if (config.has(filter_config)) + { + try + { + auto policy = std::make_shared(); + policy->setFullName(database, table_name, user_name); + policy->conditions[RowPolicy::SELECT_FILTER] = config.getString(filter_config); + policy->roles.push_back(user_name); + policies.push_back(policy); + } + catch (...) + { + tryLogCurrentException( + log, + "Could not parse row policy " + backQuote(user_name) + " on table " + backQuoteIfNeed(database) + "." + + backQuoteIfNeed(table_name)); + } + } + } + } + } + } + return policies; + } } @@ -128,6 +182,8 @@ void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfigur std::vector> all_entities; for (const auto & entity : parseQuotas(config, getLogger())) all_entities.emplace_back(generateID(*entity), entity); + for (const auto & entity : parseRowPolicies(config, getLogger())) + all_entities.emplace_back(generateID(*entity), entity); memory_storage.setAll(all_entities); } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h b/dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h index 96d97280f03..88ad9430bf0 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h @@ -24,11 +24,16 @@ struct AggregateFunctionArgMinMaxData ResultData result; // the argument at which the minimum/maximum value is reached. ValueData value; // value for which the minimum/maximum is calculated. + + static bool allocatesMemoryInArena() + { + return ResultData::allocatesMemoryInArena() || ValueData::allocatesMemoryInArena(); + } }; /// Returns the first arg value found for the minimum/maximum value. Example: argMax(arg, value). -template -class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper> +template +class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper> { private: const DataTypePtr & type_res; @@ -36,7 +41,7 @@ private: public: AggregateFunctionArgMinMax(const DataTypePtr & type_res_, const DataTypePtr & type_val_) - : IAggregateFunctionDataHelper>({type_res_, type_val_}, {}), + : IAggregateFunctionDataHelper>({type_res_, type_val_}, {}), type_res(this->argument_types[0]), type_val(this->argument_types[1]) { if (!type_val->isComparable()) @@ -77,7 +82,7 @@ public: bool allocatesMemoryInArena() const override { - return AllocatesMemoryInArena; + return Data::allocatesMemoryInArena(); } void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index a6779eb5d9e..e26f1a1f528 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -166,6 +166,11 @@ public: { return has() && assert_cast(column).getData()[row_num] == value; } + + static bool allocatesMemoryInArena() + { + return false; + } }; @@ -384,6 +389,11 @@ public: { return has() && assert_cast(column).getDataAtWithTerminatingZero(row_num) == getStringRef(); } + + static bool allocatesMemoryInArena() + { + return true; + } }; static_assert( @@ -555,6 +565,11 @@ public: { return has() && to.value == value; } + + static bool allocatesMemoryInArena() + { + return false; + } }; @@ -675,15 +690,15 @@ struct AggregateFunctionAnyHeavyData : Data }; -template -class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper> +template +class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper> { private: DataTypePtr & type; public: AggregateFunctionsSingleValue(const DataTypePtr & type_) - : IAggregateFunctionDataHelper>({type_}, {}) + : IAggregateFunctionDataHelper>({type_}, {}) , type(this->argument_types[0]) { if (StringRef(Data::name()) == StringRef("min") @@ -724,7 +739,7 @@ public: bool allocatesMemoryInArena() const override { - return use_arena; + return Data::allocatesMemoryInArena(); } void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override diff --git a/dbms/src/AggregateFunctions/HelpersMinMaxAny.h b/dbms/src/AggregateFunctions/HelpersMinMaxAny.h index 8457e33dcbb..6c027b5a6de 100644 --- a/dbms/src/AggregateFunctions/HelpersMinMaxAny.h +++ b/dbms/src/AggregateFunctions/HelpersMinMaxAny.h @@ -13,8 +13,8 @@ namespace DB { -/// min, max, any, anyLast -template