thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into enable-internal-mysql-on-non-linux

Browse Source
This commit is contained in:
Alexey Milovidov 2019-12-29 20:32:38 +03:00
commit 02c06fb156
330 changed files with 5482 additions and 7702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
contrib/arrow-cmake/CMakeLists.txt
View File

@ -102,10 +102,16 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}")
set(ARROW_IPC_SRC_DIR ${ARROW_SRC_DIR}/arrow/ipc)
set(ARROW_FORMAT_SRC_DIR ${ARROW_SRC_DIR}/../../format)
set(FLATBUFFERS_COMPILED_OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/arrow/ipc)
set(ARROW_GENERATED_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/arrow_gen_headers)
set(FLATBUFFERS_COMPILED_OUT_DIR ${ARROW_GENERATED_INCLUDE_DIR}/arrow/ipc)
set(FBS_OUTPUT_FILES "${FLATBUFFERS_COMPILED_OUT_DIR}/File_generated.h" "${FLATBUFFERS_COMPILED_OUT_DIR}/Message_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/feather_generated.h")
set(FBS_OUTPUT_FILES
"${FLATBUFFERS_COMPILED_OUT_DIR}/File_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Message_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/feather_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Schema_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/SparseTensor_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Tensor_generated.h")
set(FBS_SRC
${ARROW_FORMAT_SRC_DIR}/Message.fbs
@ -231,12 +237,10 @@ set(ARROW_SRCS
${LIBRARY_DIR}/ipc/dictionary.cc
${LIBRARY_DIR}/ipc/feather.cc
# ${LIBRARY_DIR}/ipc/file_to_stream.cc
${LIBRARY_DIR}/ipc/message.cc
${LIBRARY_DIR}/ipc/metadata_internal.cc
${LIBRARY_DIR}/ipc/options.cc
${LIBRARY_DIR}/ipc/reader.cc
# ${LIBRARY_DIR}/ipc/stream_to_file.cc
${LIBRARY_DIR}/ipc/writer.cc
${LIBRARY_DIR}/io/buffered.cc
@ -249,8 +253,6 @@ set(ARROW_SRCS
${LIBRARY_DIR}/util/basic_decimal.cc
${LIBRARY_DIR}/util/bit_util.cc
# ${LIBRARY_DIR}/util/compression_brotli.cc
# ${LIBRARY_DIR}/util/compression_bz2.cc
${LIBRARY_DIR}/util/compression.cc
${LIBRARY_DIR}/util/compression_lz4.cc
${LIBRARY_DIR}/util/compression_snappy.cc
@ -268,7 +270,6 @@ set(ARROW_SRCS
${LIBRARY_DIR}/util/task_group.cc
${LIBRARY_DIR}/util/thread_pool.cc
${LIBRARY_DIR}/util/trie.cc
# ${LIBRARY_DIR}/util/uri.cc
${LIBRARY_DIR}/util/utf8.cc
${LIBRARY_DIR}/vendored/base64.cpp
@ -356,6 +357,8 @@ target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_D
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_GENERATED_INCLUDE_DIR})
# === parquet
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/parquet)

181
contrib/arrow-cmake/cpp/src/arrow/ipc/File_generated.h
View File

@ -1,181 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_FILE_ORG_APACHE_ARROW_FLATBUF_H_
#define FLATBUFFERS_GENERATED_FILE_ORG_APACHE_ARROW_FLATBUF_H_
#include "flatbuffers/flatbuffers.h"
#include "Schema_generated.h"
namespace org {
namespace apache {
namespace arrow {
namespace flatbuf {
struct Footer;
struct Block;
FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(8) Block FLATBUFFERS_FINAL_CLASS {
private:
int64_t offset_;
int32_t metaDataLength_;
int32_t padding0__;
int64_t bodyLength_;
public:
Block() {
memset(static_cast<void *>(this), 0, sizeof(Block));
}
Block(int64_t _offset, int32_t _metaDataLength, int64_t _bodyLength)
: offset_(flatbuffers::EndianScalar(_offset)),
metaDataLength_(flatbuffers::EndianScalar(_metaDataLength)),
padding0__(0),
bodyLength_(flatbuffers::EndianScalar(_bodyLength)) {
(void)padding0__;
}
/// Index to the start of the RecordBlock (note this is past the Message header)
int64_t offset() const {
return flatbuffers::EndianScalar(offset_);
}
/// Length of the metadata
int32_t metaDataLength() const {
return flatbuffers::EndianScalar(metaDataLength_);
}
/// Length of the data (this is aligned so there can be a gap between this and
/// the metatdata).
int64_t bodyLength() const {
return flatbuffers::EndianScalar(bodyLength_);
}
};
FLATBUFFERS_STRUCT_END(Block, 24);
/// ----------------------------------------------------------------------
/// Arrow File metadata
///
struct Footer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_SCHEMA = 6,
VT_DICTIONARIES = 8,
VT_RECORDBATCHES = 10
};
MetadataVersion version() const {
return static_cast<MetadataVersion>(GetField<int16_t>(VT_VERSION, 0));
}
const Schema *schema() const {
return GetPointer<const Schema *>(VT_SCHEMA);
}
const flatbuffers::Vector<const Block *> *dictionaries() const {
return GetPointer<const flatbuffers::Vector<const Block *> *>(VT_DICTIONARIES);
}
const flatbuffers::Vector<const Block *> *recordBatches() const {
return GetPointer<const flatbuffers::Vector<const Block *> *>(VT_RECORDBATCHES);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int16_t>(verifier, VT_VERSION) &&
VerifyOffset(verifier, VT_SCHEMA) &&
verifier.VerifyTable(schema()) &&
VerifyOffset(verifier, VT_DICTIONARIES) &&
verifier.VerifyVector(dictionaries()) &&
VerifyOffset(verifier, VT_RECORDBATCHES) &&
verifier.VerifyVector(recordBatches()) &&
verifier.EndTable();
}
};
struct FooterBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(MetadataVersion version) {
fbb_.AddElement<int16_t>(Footer::VT_VERSION, static_cast<int16_t>(version), 0);
}
void add_schema(flatbuffers::Offset<Schema> schema) {
fbb_.AddOffset(Footer::VT_SCHEMA, schema);
}
void add_dictionaries(flatbuffers::Offset<flatbuffers::Vector<const Block *>> dictionaries) {
fbb_.AddOffset(Footer::VT_DICTIONARIES, dictionaries);
}
void add_recordBatches(flatbuffers::Offset<flatbuffers::Vector<const Block *>> recordBatches) {
fbb_.AddOffset(Footer::VT_RECORDBATCHES, recordBatches);
}
explicit FooterBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
FooterBuilder &operator=(const FooterBuilder &);
flatbuffers::Offset<Footer> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Footer>(end);
return o;
}
};
inline flatbuffers::Offset<Footer> CreateFooter(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion_V1,
flatbuffers::Offset<Schema> schema = 0,
flatbuffers::Offset<flatbuffers::Vector<const Block *>> dictionaries = 0,
flatbuffers::Offset<flatbuffers::Vector<const Block *>> recordBatches = 0) {
FooterBuilder builder_(_fbb);
builder_.add_recordBatches(recordBatches);
builder_.add_dictionaries(dictionaries);
builder_.add_schema(schema);
builder_.add_version(version);
return builder_.Finish();
}
inline flatbuffers::Offset<Footer> CreateFooterDirect(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion_V1,
flatbuffers::Offset<Schema> schema = 0,
const std::vector<Block> *dictionaries = nullptr,
const std::vector<Block> *recordBatches = nullptr) {
auto dictionaries__ = dictionaries ? _fbb.CreateVectorOfStructs<Block>(*dictionaries) : 0;
auto recordBatches__ = recordBatches ? _fbb.CreateVectorOfStructs<Block>(*recordBatches) : 0;
return org::apache::arrow::flatbuf::CreateFooter(
_fbb,
version,
schema,
dictionaries__,
recordBatches__);
}
inline const org::apache::arrow::flatbuf::Footer *GetFooter(const void *buf) {
return flatbuffers::GetRoot<org::apache::arrow::flatbuf::Footer>(buf);
}
inline const org::apache::arrow::flatbuf::Footer *GetSizePrefixedFooter(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<org::apache::arrow::flatbuf::Footer>(buf);
}
inline bool VerifyFooterBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<org::apache::arrow::flatbuf::Footer>(nullptr);
}
inline bool VerifySizePrefixedFooterBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<org::apache::arrow::flatbuf::Footer>(nullptr);
}
inline void FinishFooterBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Footer> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedFooterBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Footer> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace flatbuf
} // namespace arrow
} // namespace apache
} // namespace org
#endif // FLATBUFFERS_GENERATED_FILE_ORG_APACHE_ARROW_FLATBUF_H_

508
contrib/arrow-cmake/cpp/src/arrow/ipc/Message_generated.h
View File

@ -1,508 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_MESSAGE_ORG_APACHE_ARROW_FLATBUF_H_
#define FLATBUFFERS_GENERATED_MESSAGE_ORG_APACHE_ARROW_FLATBUF_H_
#include "flatbuffers/flatbuffers.h"
#include "Schema_generated.h"
#include "SparseTensor_generated.h"
#include "Tensor_generated.h"
namespace org {
namespace apache {
namespace arrow {
namespace flatbuf {
struct FieldNode;
struct RecordBatch;
struct DictionaryBatch;
struct Message;
/// ----------------------------------------------------------------------
/// The root Message type
/// This union enables us to easily send different message types without
/// redundant storage, and in the future we can easily add new message types.
///
/// Arrow implementations do not need to implement all of the message types,
/// which may include experimental metadata types. For maximum compatibility,
/// it is best to send data using RecordBatch
enum MessageHeader {
MessageHeader_NONE = 0,
MessageHeader_Schema = 1,
MessageHeader_DictionaryBatch = 2,
MessageHeader_RecordBatch = 3,
MessageHeader_Tensor = 4,
MessageHeader_SparseTensor = 5,
MessageHeader_MIN = MessageHeader_NONE,
MessageHeader_MAX = MessageHeader_SparseTensor
};
inline const MessageHeader (&EnumValuesMessageHeader())[6] {
static const MessageHeader values[] = {
MessageHeader_NONE,
MessageHeader_Schema,
MessageHeader_DictionaryBatch,
MessageHeader_RecordBatch,
MessageHeader_Tensor,
MessageHeader_SparseTensor
};
return values;
}
inline const char * const *EnumNamesMessageHeader() {
static const char * const names[] = {
"NONE",
"Schema",
"DictionaryBatch",
"RecordBatch",
"Tensor",
"SparseTensor",
nullptr
};
return names;
}
inline const char *EnumNameMessageHeader(MessageHeader e) {
if (e < MessageHeader_NONE || e > MessageHeader_SparseTensor) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesMessageHeader()[index];
}
template<typename T> struct MessageHeaderTraits {
static const MessageHeader enum_value = MessageHeader_NONE;
};
template<> struct MessageHeaderTraits<Schema> {
static const MessageHeader enum_value = MessageHeader_Schema;
};
template<> struct MessageHeaderTraits<DictionaryBatch> {
static const MessageHeader enum_value = MessageHeader_DictionaryBatch;
};
template<> struct MessageHeaderTraits<RecordBatch> {
static const MessageHeader enum_value = MessageHeader_RecordBatch;
};
template<> struct MessageHeaderTraits<Tensor> {
static const MessageHeader enum_value = MessageHeader_Tensor;
};
template<> struct MessageHeaderTraits<SparseTensor> {
static const MessageHeader enum_value = MessageHeader_SparseTensor;
};
bool VerifyMessageHeader(flatbuffers::Verifier &verifier, const void *obj, MessageHeader type);
bool VerifyMessageHeaderVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);
/// ----------------------------------------------------------------------
/// Data structures for describing a table row batch (a collection of
/// equal-length Arrow arrays)
/// Metadata about a field at some level of a nested type tree (but not
/// its children).
///
/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
/// null_count: 0} for its Int16 node, as separate FieldNode structs
FLATBUFFERS_MANUALLY_ALIGNED_STRUCT(8) FieldNode FLATBUFFERS_FINAL_CLASS {
private:
int64_t length_;
int64_t null_count_;
public:
FieldNode() {
memset(static_cast<void *>(this), 0, sizeof(FieldNode));
}
FieldNode(int64_t _length, int64_t _null_count)
: length_(flatbuffers::EndianScalar(_length)),
null_count_(flatbuffers::EndianScalar(_null_count)) {
}
/// The number of value slots in the Arrow array at this level of a nested
/// tree
int64_t length() const {
return flatbuffers::EndianScalar(length_);
}
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
int64_t null_count() const {
return flatbuffers::EndianScalar(null_count_);
}
};
FLATBUFFERS_STRUCT_END(FieldNode, 16);
/// A data header describing the shared memory layout of a "record" or "row"
/// batch. Some systems call this a "row batch" internally and others a "record
/// batch".
struct RecordBatch FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_LENGTH = 4,
VT_NODES = 6,
VT_BUFFERS = 8
};
/// number of records / rows. The arrays in the batch should all have this
/// length
int64_t length() const {
return GetField<int64_t>(VT_LENGTH, 0);
}
/// Nodes correspond to the pre-ordered flattened logical schema
const flatbuffers::Vector<const FieldNode *> *nodes() const {
return GetPointer<const flatbuffers::Vector<const FieldNode *> *>(VT_NODES);
}
/// Buffers correspond to the pre-ordered flattened buffer tree
///
/// The number of buffers appended to this list depends on the schema. For
/// example, most primitive arrays will have 2 buffers, 1 for the validity
/// bitmap and 1 for the values. For struct arrays, there will only be a
/// single buffer for the validity (nulls) bitmap
const flatbuffers::Vector<const Buffer *> *buffers() const {
return GetPointer<const flatbuffers::Vector<const Buffer *> *>(VT_BUFFERS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int64_t>(verifier, VT_LENGTH) &&
VerifyOffset(verifier, VT_NODES) &&
verifier.VerifyVector(nodes()) &&
VerifyOffset(verifier, VT_BUFFERS) &&
verifier.VerifyVector(buffers()) &&
verifier.EndTable();
}
};
struct RecordBatchBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_length(int64_t length) {
fbb_.AddElement<int64_t>(RecordBatch::VT_LENGTH, length, 0);
}
void add_nodes(flatbuffers::Offset<flatbuffers::Vector<const FieldNode *>> nodes) {
fbb_.AddOffset(RecordBatch::VT_NODES, nodes);
}
void add_buffers(flatbuffers::Offset<flatbuffers::Vector<const Buffer *>> buffers) {
fbb_.AddOffset(RecordBatch::VT_BUFFERS, buffers);
}
explicit RecordBatchBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
RecordBatchBuilder &operator=(const RecordBatchBuilder &);
flatbuffers::Offset<RecordBatch> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RecordBatch>(end);
return o;
}
};
inline flatbuffers::Offset<RecordBatch> CreateRecordBatch(
flatbuffers::FlatBufferBuilder &_fbb,
int64_t length = 0,
flatbuffers::Offset<flatbuffers::Vector<const FieldNode *>> nodes = 0,
flatbuffers::Offset<flatbuffers::Vector<const Buffer *>> buffers = 0) {
RecordBatchBuilder builder_(_fbb);
builder_.add_length(length);
builder_.add_buffers(buffers);
builder_.add_nodes(nodes);
return builder_.Finish();
}
inline flatbuffers::Offset<RecordBatch> CreateRecordBatchDirect(
flatbuffers::FlatBufferBuilder &_fbb,
int64_t length = 0,
const std::vector<FieldNode> *nodes = nullptr,
const std::vector<Buffer> *buffers = nullptr) {
auto nodes__ = nodes ? _fbb.CreateVectorOfStructs<FieldNode>(*nodes) : 0;
auto buffers__ = buffers ? _fbb.CreateVectorOfStructs<Buffer>(*buffers) : 0;
return org::apache::arrow::flatbuf::CreateRecordBatch(
_fbb,
length,
nodes__,
buffers__);
}
/// For sending dictionary encoding information. Any Field can be
/// dictionary-encoded, but in this case none of its children may be
/// dictionary-encoded.
/// There is one vector / column per dictionary, but that vector / column
/// may be spread across multiple dictionary batches by using the isDelta
/// flag
struct DictionaryBatch FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_ID = 4,
VT_DATA = 6,
VT_ISDELTA = 8
};
int64_t id() const {
return GetField<int64_t>(VT_ID, 0);
}
const RecordBatch *data() const {
return GetPointer<const RecordBatch *>(VT_DATA);
}
/// If isDelta is true the values in the dictionary are to be appended to a
/// dictionary with the indicated id
bool isDelta() const {
return GetField<uint8_t>(VT_ISDELTA, 0) != 0;
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int64_t>(verifier, VT_ID) &&
VerifyOffset(verifier, VT_DATA) &&
verifier.VerifyTable(data()) &&
VerifyField<uint8_t>(verifier, VT_ISDELTA) &&
verifier.EndTable();
}
};
struct DictionaryBatchBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_id(int64_t id) {
fbb_.AddElement<int64_t>(DictionaryBatch::VT_ID, id, 0);
}
void add_data(flatbuffers::Offset<RecordBatch> data) {
fbb_.AddOffset(DictionaryBatch::VT_DATA, data);
}
void add_isDelta(bool isDelta) {
fbb_.AddElement<uint8_t>(DictionaryBatch::VT_ISDELTA, static_cast<uint8_t>(isDelta), 0);
}
explicit DictionaryBatchBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
DictionaryBatchBuilder &operator=(const DictionaryBatchBuilder &);
flatbuffers::Offset<DictionaryBatch> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<DictionaryBatch>(end);
return o;
}
};
inline flatbuffers::Offset<DictionaryBatch> CreateDictionaryBatch(
flatbuffers::FlatBufferBuilder &_fbb,
int64_t id = 0,
flatbuffers::Offset<RecordBatch> data = 0,
bool isDelta = false) {
DictionaryBatchBuilder builder_(_fbb);
builder_.add_id(id);
builder_.add_data(data);
builder_.add_isDelta(isDelta);
return builder_.Finish();
}
struct Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_HEADER_TYPE = 6,
VT_HEADER = 8,
VT_BODYLENGTH = 10,
VT_CUSTOM_METADATA = 12
};
MetadataVersion version() const {
return static_cast<MetadataVersion>(GetField<int16_t>(VT_VERSION, 0));
}
MessageHeader header_type() const {
return static_cast<MessageHeader>(GetField<uint8_t>(VT_HEADER_TYPE, 0));
}
const void *header() const {
return GetPointer<const void *>(VT_HEADER);
}
template<typename T> const T *header_as() const;
const Schema *header_as_Schema() const {
return header_type() == MessageHeader_Schema ? static_cast<const Schema *>(header()) : nullptr;
}
const DictionaryBatch *header_as_DictionaryBatch() const {
return header_type() == MessageHeader_DictionaryBatch ? static_cast<const DictionaryBatch *>(header()) : nullptr;
}
const RecordBatch *header_as_RecordBatch() const {
return header_type() == MessageHeader_RecordBatch ? static_cast<const RecordBatch *>(header()) : nullptr;
}
const Tensor *header_as_Tensor() const {
return header_type() == MessageHeader_Tensor ? static_cast<const Tensor *>(header()) : nullptr;
}
const SparseTensor *header_as_SparseTensor() const {
return header_type() == MessageHeader_SparseTensor ? static_cast<const SparseTensor *>(header()) : nullptr;
}
int64_t bodyLength() const {
return GetField<int64_t>(VT_BODYLENGTH, 0);
}
const flatbuffers::Vector<flatbuffers::Offset<KeyValue>> *custom_metadata() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<KeyValue>> *>(VT_CUSTOM_METADATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int16_t>(verifier, VT_VERSION) &&
VerifyField<uint8_t>(verifier, VT_HEADER_TYPE) &&
VerifyOffset(verifier, VT_HEADER) &&
VerifyMessageHeader(verifier, header(), header_type()) &&
VerifyField<int64_t>(verifier, VT_BODYLENGTH) &&
VerifyOffset(verifier, VT_CUSTOM_METADATA) &&
verifier.VerifyVector(custom_metadata()) &&
verifier.VerifyVectorOfTables(custom_metadata()) &&
verifier.EndTable();
}
};
template<> inline const Schema *Message::header_as<Schema>() const {
return header_as_Schema();
}
template<> inline const DictionaryBatch *Message::header_as<DictionaryBatch>() const {
return header_as_DictionaryBatch();
}
template<> inline const RecordBatch *Message::header_as<RecordBatch>() const {
return header_as_RecordBatch();
}
template<> inline const Tensor *Message::header_as<Tensor>() const {
return header_as_Tensor();
}
template<> inline const SparseTensor *Message::header_as<SparseTensor>() const {
return header_as_SparseTensor();
}
struct MessageBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(MetadataVersion version) {
fbb_.AddElement<int16_t>(Message::VT_VERSION, static_cast<int16_t>(version), 0);
}
void add_header_type(MessageHeader header_type) {
fbb_.AddElement<uint8_t>(Message::VT_HEADER_TYPE, static_cast<uint8_t>(header_type), 0);
}
void add_header(flatbuffers::Offset<void> header) {
fbb_.AddOffset(Message::VT_HEADER, header);
}
void add_bodyLength(int64_t bodyLength) {
fbb_.AddElement<int64_t>(Message::VT_BODYLENGTH, bodyLength, 0);
}
void add_custom_metadata(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<KeyValue>>> custom_metadata) {
fbb_.AddOffset(Message::VT_CUSTOM_METADATA, custom_metadata);
}
explicit MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
MessageBuilder &operator=(const MessageBuilder &);
flatbuffers::Offset<Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Message>(end);
return o;
}
};
inline flatbuffers::Offset<Message> CreateMessage(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion_V1,
MessageHeader header_type = MessageHeader_NONE,
flatbuffers::Offset<void> header = 0,
int64_t bodyLength = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<KeyValue>>> custom_metadata = 0) {
MessageBuilder builder_(_fbb);
builder_.add_bodyLength(bodyLength);
builder_.add_custom_metadata(custom_metadata);
builder_.add_header(header);
builder_.add_version(version);
builder_.add_header_type(header_type);
return builder_.Finish();
}
inline flatbuffers::Offset<Message> CreateMessageDirect(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion_V1,
MessageHeader header_type = MessageHeader_NONE,
flatbuffers::Offset<void> header = 0,
int64_t bodyLength = 0,
const std::vector<flatbuffers::Offset<KeyValue>> *custom_metadata = nullptr) {
auto custom_metadata__ = custom_metadata ? _fbb.CreateVector<flatbuffers::Offset<KeyValue>>(*custom_metadata) : 0;
return org::apache::arrow::flatbuf::CreateMessage(
_fbb,
version,
header_type,
header,
bodyLength,
custom_metadata__);
}
inline bool VerifyMessageHeader(flatbuffers::Verifier &verifier, const void *obj, MessageHeader type) {
switch (type) {
case MessageHeader_NONE: {
return true;
}
case MessageHeader_Schema: {
auto ptr = reinterpret_cast<const Schema *>(obj);
return verifier.VerifyTable(ptr);
}
case MessageHeader_DictionaryBatch: {
auto ptr = reinterpret_cast<const DictionaryBatch *>(obj);
return verifier.VerifyTable(ptr);
}
case MessageHeader_RecordBatch: {
auto ptr = reinterpret_cast<const RecordBatch *>(obj);
return verifier.VerifyTable(ptr);
}
case MessageHeader_Tensor: {
auto ptr = reinterpret_cast<const Tensor *>(obj);
return verifier.VerifyTable(ptr);
}
case MessageHeader_SparseTensor: {
auto ptr = reinterpret_cast<const SparseTensor *>(obj);
return verifier.VerifyTable(ptr);
}
default: return false;
}
}
inline bool VerifyMessageHeaderVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types) {
if (!values || !types) return !values && !types;
if (values->size() != types->size()) return false;
for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {
if (!VerifyMessageHeader(
verifier, values->Get(i), types->GetEnum<MessageHeader>(i))) {
return false;
}
}
return true;
}
inline const org::apache::arrow::flatbuf::Message *GetMessage(const void *buf) {
return flatbuffers::GetRoot<org::apache::arrow::flatbuf::Message>(buf);
}
inline const org::apache::arrow::flatbuf::Message *GetSizePrefixedMessage(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<org::apache::arrow::flatbuf::Message>(buf);
}
inline bool VerifyMessageBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<org::apache::arrow::flatbuf::Message>(nullptr);
}
inline bool VerifySizePrefixedMessageBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<org::apache::arrow::flatbuf::Message>(nullptr);
}
inline void FinishMessageBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Message> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedMessageBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Message> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace flatbuf
} // namespace arrow
} // namespace apache
} // namespace org
#endif // FLATBUFFERS_GENERATED_MESSAGE_ORG_APACHE_ARROW_FLATBUF_H_

2045
contrib/arrow-cmake/cpp/src/arrow/ipc/Schema_generated.h
View File

File diff suppressed because it is too large Load Diff

644
contrib/arrow-cmake/cpp/src/arrow/ipc/SparseTensor_generated.h
View File

@ -1,644 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_SPARSETENSOR_ORG_APACHE_ARROW_FLATBUF_H_
#define FLATBUFFERS_GENERATED_SPARSETENSOR_ORG_APACHE_ARROW_FLATBUF_H_
#include "flatbuffers/flatbuffers.h"
#include "Schema_generated.h"
#include "Tensor_generated.h"
namespace org {
namespace apache {
namespace arrow {
namespace flatbuf {
struct SparseTensorIndexCOO;
struct SparseMatrixIndexCSR;
struct SparseTensor;
enum SparseTensorIndex {
SparseTensorIndex_NONE = 0,
SparseTensorIndex_SparseTensorIndexCOO = 1,
SparseTensorIndex_SparseMatrixIndexCSR = 2,
SparseTensorIndex_MIN = SparseTensorIndex_NONE,
SparseTensorIndex_MAX = SparseTensorIndex_SparseMatrixIndexCSR
};
inline const SparseTensorIndex (&EnumValuesSparseTensorIndex())[3] {
static const SparseTensorIndex values[] = {
SparseTensorIndex_NONE,
SparseTensorIndex_SparseTensorIndexCOO,
SparseTensorIndex_SparseMatrixIndexCSR
};
return values;
}
inline const char * const *EnumNamesSparseTensorIndex() {
static const char * const names[] = {
"NONE",
"SparseTensorIndexCOO",
"SparseMatrixIndexCSR",
nullptr
};
return names;
}
inline const char *EnumNameSparseTensorIndex(SparseTensorIndex e) {
if (e < SparseTensorIndex_NONE || e > SparseTensorIndex_SparseMatrixIndexCSR) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesSparseTensorIndex()[index];
}
template<typename T> struct SparseTensorIndexTraits {
static const SparseTensorIndex enum_value = SparseTensorIndex_NONE;
};
template<> struct SparseTensorIndexTraits<SparseTensorIndexCOO> {
static const SparseTensorIndex enum_value = SparseTensorIndex_SparseTensorIndexCOO;
};
template<> struct SparseTensorIndexTraits<SparseMatrixIndexCSR> {
static const SparseTensorIndex enum_value = SparseTensorIndex_SparseMatrixIndexCSR;
};
bool VerifySparseTensorIndex(flatbuffers::Verifier &verifier, const void *obj, SparseTensorIndex type);
bool VerifySparseTensorIndexVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);
/// ----------------------------------------------------------------------
/// EXPERIMENTAL: Data structures for sparse tensors
/// Coodinate (COO) format of sparse tensor index.
///
/// COO's index list are represented as a NxM matrix,
/// where N is the number of non-zero values,
/// and M is the number of dimensions of a sparse tensor.
///
/// indicesBuffer stores the location and size of the data of this indices
/// matrix. The value type and the stride of the indices matrix is
/// specified in indicesType and indicesStrides fields.
///
/// For example, let X be a 2x3x4x5 tensor, and it has the following
/// 6 non-zero values:
///
/// X[0, 1, 2, 0] := 1
/// X[1, 1, 2, 3] := 2
/// X[0, 2, 1, 0] := 3
/// X[0, 1, 3, 0] := 4
/// X[0, 1, 2, 1] := 5
/// X[1, 2, 0, 4] := 6
///
/// In COO format, the index matrix of X is the following 4x6 matrix:
///
/// [[0, 0, 0, 0, 1, 1],
/// [1, 1, 1, 2, 1, 2],
/// [2, 2, 3, 1, 2, 0],
/// [0, 1, 0, 0, 3, 4]]
///
/// Note that the indices are sorted in lexicographical order.
struct SparseTensorIndexCOO FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_INDICESTYPE = 4,
VT_INDICESSTRIDES = 6,
VT_INDICESBUFFER = 8
};
/// The type of values in indicesBuffer
const Int *indicesType() const {
return GetPointer<const Int *>(VT_INDICESTYPE);
}
/// Non-negative byte offsets to advance one value cell along each dimension
const flatbuffers::Vector<int64_t> *indicesStrides() const {
return GetPointer<const flatbuffers::Vector<int64_t> *>(VT_INDICESSTRIDES);
}
/// The location and size of the indices matrix's data
const Buffer *indicesBuffer() const {
return GetStruct<const Buffer *>(VT_INDICESBUFFER);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_INDICESTYPE) &&
verifier.VerifyTable(indicesType()) &&
VerifyOffset(verifier, VT_INDICESSTRIDES) &&
verifier.VerifyVector(indicesStrides()) &&
VerifyField<Buffer>(verifier, VT_INDICESBUFFER) &&
verifier.EndTable();
}
};
struct SparseTensorIndexCOOBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_indicesType(flatbuffers::Offset<Int> indicesType) {
fbb_.AddOffset(SparseTensorIndexCOO::VT_INDICESTYPE, indicesType);
}
void add_indicesStrides(flatbuffers::Offset<flatbuffers::Vector<int64_t>> indicesStrides) {
fbb_.AddOffset(SparseTensorIndexCOO::VT_INDICESSTRIDES, indicesStrides);
}
void add_indicesBuffer(const Buffer *indicesBuffer) {
fbb_.AddStruct(SparseTensorIndexCOO::VT_INDICESBUFFER, indicesBuffer);
}
explicit SparseTensorIndexCOOBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
SparseTensorIndexCOOBuilder &operator=(const SparseTensorIndexCOOBuilder &);
flatbuffers::Offset<SparseTensorIndexCOO> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<SparseTensorIndexCOO>(end);
return o;
}
};
inline flatbuffers::Offset<SparseTensorIndexCOO> CreateSparseTensorIndexCOO(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<Int> indicesType = 0,
flatbuffers::Offset<flatbuffers::Vector<int64_t>> indicesStrides = 0,
const Buffer *indicesBuffer = 0) {
SparseTensorIndexCOOBuilder builder_(_fbb);
builder_.add_indicesBuffer(indicesBuffer);
builder_.add_indicesStrides(indicesStrides);
builder_.add_indicesType(indicesType);
return builder_.Finish();
}
inline flatbuffers::Offset<SparseTensorIndexCOO> CreateSparseTensorIndexCOODirect(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<Int> indicesType = 0,
const std::vector<int64_t> *indicesStrides = nullptr,
const Buffer *indicesBuffer = 0) {
auto indicesStrides__ = indicesStrides ? _fbb.CreateVector<int64_t>(*indicesStrides) : 0;
return org::apache::arrow::flatbuf::CreateSparseTensorIndexCOO(
_fbb,
indicesType,
indicesStrides__,
indicesBuffer);
}
/// Compressed Sparse Row format, that is matrix-specific.
struct SparseMatrixIndexCSR FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_INDPTRTYPE = 4,
VT_INDPTRBUFFER = 6,
VT_INDICESTYPE = 8,
VT_INDICESBUFFER = 10
};
/// The type of values in indptrBuffer
const Int *indptrType() const {
return GetPointer<const Int *>(VT_INDPTRTYPE);
}
/// indptrBuffer stores the location and size of indptr array that
/// represents the range of the rows.
/// The i-th row spans from indptr[i] to indptr[i+1] in the data.
/// The length of this array is 1 + (the number of rows), and the type
/// of index value is long.
///
/// For example, let X be the following 6x4 matrix:
///
/// X := [[0, 1, 2, 0],
/// [0, 0, 3, 0],
/// [0, 4, 0, 5],
/// [0, 0, 0, 0],
/// [6, 0, 7, 8],
/// [0, 9, 0, 0]].
///
/// The array of non-zero values in X is:
///
/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
///
/// And the indptr of X is:
///
/// indptr(X) = [0, 2, 3, 5, 5, 8, 10].
const Buffer *indptrBuffer() const {
return GetStruct<const Buffer *>(VT_INDPTRBUFFER);
}
/// The type of values in indicesBuffer
const Int *indicesType() const {
return GetPointer<const Int *>(VT_INDICESTYPE);
}
/// indicesBuffer stores the location and size of the array that
/// contains the column indices of the corresponding non-zero values.
/// The type of index value is long.
///
/// For example, the indices of the above X is:
///
/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
///
/// Note that the indices are sorted in lexicographical order for each row.
const Buffer *indicesBuffer() const {
return GetStruct<const Buffer *>(VT_INDICESBUFFER);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_INDPTRTYPE) &&
verifier.VerifyTable(indptrType()) &&
VerifyField<Buffer>(verifier, VT_INDPTRBUFFER) &&
VerifyOffset(verifier, VT_INDICESTYPE) &&
verifier.VerifyTable(indicesType()) &&
VerifyField<Buffer>(verifier, VT_INDICESBUFFER) &&
verifier.EndTable();
}
};
struct SparseMatrixIndexCSRBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_indptrType(flatbuffers::Offset<Int> indptrType) {
fbb_.AddOffset(SparseMatrixIndexCSR::VT_INDPTRTYPE, indptrType);
}
void add_indptrBuffer(const Buffer *indptrBuffer) {
fbb_.AddStruct(SparseMatrixIndexCSR::VT_INDPTRBUFFER, indptrBuffer);
}
void add_indicesType(flatbuffers::Offset<Int> indicesType) {
fbb_.AddOffset(SparseMatrixIndexCSR::VT_INDICESTYPE, indicesType);
}
void add_indicesBuffer(const Buffer *indicesBuffer) {
fbb_.AddStruct(SparseMatrixIndexCSR::VT_INDICESBUFFER, indicesBuffer);
}
explicit SparseMatrixIndexCSRBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
SparseMatrixIndexCSRBuilder &operator=(const SparseMatrixIndexCSRBuilder &);
flatbuffers::Offset<SparseMatrixIndexCSR> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<SparseMatrixIndexCSR>(end);
return o;
}
};
inline flatbuffers::Offset<SparseMatrixIndexCSR> CreateSparseMatrixIndexCSR(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<Int> indptrType = 0,
const Buffer *indptrBuffer = 0,
flatbuffers::Offset<Int> indicesType = 0,
const Buffer *indicesBuffer = 0) {
SparseMatrixIndexCSRBuilder builder_(_fbb);
builder_.add_indicesBuffer(indicesBuffer);
builder_.add_indicesType(indicesType);
builder_.add_indptrBuffer(indptrBuffer);
builder_.add_indptrType(indptrType);
return builder_.Finish();
}
struct SparseTensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_TYPE_TYPE = 4,
VT_TYPE = 6,
VT_SHAPE = 8,
VT_NON_ZERO_LENGTH = 10,
VT_SPARSEINDEX_TYPE = 12,
VT_SPARSEINDEX = 14,
VT_DATA = 16
};
Type type_type() const {
return static_cast<Type>(GetField<uint8_t>(VT_TYPE_TYPE, 0));
}
/// The type of data contained in a value cell.
/// Currently only fixed-width value types are supported,
/// no strings or nested types.
const void *type() const {
return GetPointer<const void *>(VT_TYPE);
}
template<typename T> const T *type_as() const;
const Null *type_as_Null() const {
return type_type() == Type_Null ? static_cast<const Null *>(type()) : nullptr;
}
const Int *type_as_Int() const {
return type_type() == Type_Int ? static_cast<const Int *>(type()) : nullptr;
}
const FloatingPoint *type_as_FloatingPoint() const {
return type_type() == Type_FloatingPoint ? static_cast<const FloatingPoint *>(type()) : nullptr;
}
const Binary *type_as_Binary() const {
return type_type() == Type_Binary ? static_cast<const Binary *>(type()) : nullptr;
}
const Utf8 *type_as_Utf8() const {
return type_type() == Type_Utf8 ? static_cast<const Utf8 *>(type()) : nullptr;
}
const Bool *type_as_Bool() const {
return type_type() == Type_Bool ? static_cast<const Bool *>(type()) : nullptr;
}
const Decimal *type_as_Decimal() const {
return type_type() == Type_Decimal ? static_cast<const Decimal *>(type()) : nullptr;
}
const Date *type_as_Date() const {
return type_type() == Type_Date ? static_cast<const Date *>(type()) : nullptr;
}
const Time *type_as_Time() const {
return type_type() == Type_Time ? static_cast<const Time *>(type()) : nullptr;
}
const Timestamp *type_as_Timestamp() const {
return type_type() == Type_Timestamp ? static_cast<const Timestamp *>(type()) : nullptr;
}
const Interval *type_as_Interval() const {
return type_type() == Type_Interval ? static_cast<const Interval *>(type()) : nullptr;
}
const List *type_as_List() const {
return type_type() == Type_List ? static_cast<const List *>(type()) : nullptr;
}
const Struct_ *type_as_Struct_() const {
return type_type() == Type_Struct_ ? static_cast<const Struct_ *>(type()) : nullptr;
}
const Union *type_as_Union() const {
return type_type() == Type_Union ? static_cast<const Union *>(type()) : nullptr;
}
const FixedSizeBinary *type_as_FixedSizeBinary() const {
return type_type() == Type_FixedSizeBinary ? static_cast<const FixedSizeBinary *>(type()) : nullptr;
}
const FixedSizeList *type_as_FixedSizeList() const {
return type_type() == Type_FixedSizeList ? static_cast<const FixedSizeList *>(type()) : nullptr;
}
const Map *type_as_Map() const {
return type_type() == Type_Map ? static_cast<const Map *>(type()) : nullptr;
}
const Duration *type_as_Duration() const {
return type_type() == Type_Duration ? static_cast<const Duration *>(type()) : nullptr;
}
const LargeBinary *type_as_LargeBinary() const {
return type_type() == Type_LargeBinary ? static_cast<const LargeBinary *>(type()) : nullptr;
}
const LargeUtf8 *type_as_LargeUtf8() const {
return type_type() == Type_LargeUtf8 ? static_cast<const LargeUtf8 *>(type()) : nullptr;
}
const LargeList *type_as_LargeList() const {
return type_type() == Type_LargeList ? static_cast<const LargeList *>(type()) : nullptr;
}
/// The dimensions of the tensor, optionally named.
const flatbuffers::Vector<flatbuffers::Offset<TensorDim>> *shape() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<TensorDim>> *>(VT_SHAPE);
}
/// The number of non-zero values in a sparse tensor.
int64_t non_zero_length() const {
return GetField<int64_t>(VT_NON_ZERO_LENGTH, 0);
}
SparseTensorIndex sparseIndex_type() const {
return static_cast<SparseTensorIndex>(GetField<uint8_t>(VT_SPARSEINDEX_TYPE, 0));
}
/// Sparse tensor index
const void *sparseIndex() const {
return GetPointer<const void *>(VT_SPARSEINDEX);
}
template<typename T> const T *sparseIndex_as() const;
const SparseTensorIndexCOO *sparseIndex_as_SparseTensorIndexCOO() const {
return sparseIndex_type() == SparseTensorIndex_SparseTensorIndexCOO ? static_cast<const SparseTensorIndexCOO *>(sparseIndex()) : nullptr;
}
const SparseMatrixIndexCSR *sparseIndex_as_SparseMatrixIndexCSR() const {
return sparseIndex_type() == SparseTensorIndex_SparseMatrixIndexCSR ? static_cast<const SparseMatrixIndexCSR *>(sparseIndex()) : nullptr;
}
/// The location and size of the tensor's data
const Buffer *data() const {
return GetStruct<const Buffer *>(VT_DATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<uint8_t>(verifier, VT_TYPE_TYPE) &&
VerifyOffset(verifier, VT_TYPE) &&
VerifyType(verifier, type(), type_type()) &&
VerifyOffset(verifier, VT_SHAPE) &&
verifier.VerifyVector(shape()) &&
verifier.VerifyVectorOfTables(shape()) &&
VerifyField<int64_t>(verifier, VT_NON_ZERO_LENGTH) &&
VerifyField<uint8_t>(verifier, VT_SPARSEINDEX_TYPE) &&
VerifyOffset(verifier, VT_SPARSEINDEX) &&
VerifySparseTensorIndex(verifier, sparseIndex(), sparseIndex_type()) &&
VerifyField<Buffer>(verifier, VT_DATA) &&
verifier.EndTable();
}
};
template<> inline const Null *SparseTensor::type_as<Null>() const {
return type_as_Null();
}
template<> inline const Int *SparseTensor::type_as<Int>() const {
return type_as_Int();
}
template<> inline const FloatingPoint *SparseTensor::type_as<FloatingPoint>() const {
return type_as_FloatingPoint();
}
template<> inline const Binary *SparseTensor::type_as<Binary>() const {
return type_as_Binary();
}
template<> inline const Utf8 *SparseTensor::type_as<Utf8>() const {
return type_as_Utf8();
}
template<> inline const Bool *SparseTensor::type_as<Bool>() const {
return type_as_Bool();
}
template<> inline const Decimal *SparseTensor::type_as<Decimal>() const {
return type_as_Decimal();
}
template<> inline const Date *SparseTensor::type_as<Date>() const {
return type_as_Date();
}
template<> inline const Time *SparseTensor::type_as<Time>() const {
return type_as_Time();
}
template<> inline const Timestamp *SparseTensor::type_as<Timestamp>() const {
return type_as_Timestamp();
}
template<> inline const Interval *SparseTensor::type_as<Interval>() const {
return type_as_Interval();
}
template<> inline const List *SparseTensor::type_as<List>() const {
return type_as_List();
}
template<> inline const Struct_ *SparseTensor::type_as<Struct_>() const {
return type_as_Struct_();
}
template<> inline const Union *SparseTensor::type_as<Union>() const {
return type_as_Union();
}
template<> inline const FixedSizeBinary *SparseTensor::type_as<FixedSizeBinary>() const {
return type_as_FixedSizeBinary();
}
template<> inline const FixedSizeList *SparseTensor::type_as<FixedSizeList>() const {
return type_as_FixedSizeList();
}
template<> inline const Map *SparseTensor::type_as<Map>() const {
return type_as_Map();
}
template<> inline const Duration *SparseTensor::type_as<Duration>() const {
return type_as_Duration();
}
template<> inline const LargeBinary *SparseTensor::type_as<LargeBinary>() const {
return type_as_LargeBinary();
}
template<> inline const LargeUtf8 *SparseTensor::type_as<LargeUtf8>() const {
return type_as_LargeUtf8();
}
template<> inline const LargeList *SparseTensor::type_as<LargeList>() const {
return type_as_LargeList();
}
template<> inline const SparseTensorIndexCOO *SparseTensor::sparseIndex_as<SparseTensorIndexCOO>() const {
return sparseIndex_as_SparseTensorIndexCOO();
}
template<> inline const SparseMatrixIndexCSR *SparseTensor::sparseIndex_as<SparseMatrixIndexCSR>() const {
return sparseIndex_as_SparseMatrixIndexCSR();
}
struct SparseTensorBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_type_type(Type type_type) {
fbb_.AddElement<uint8_t>(SparseTensor::VT_TYPE_TYPE, static_cast<uint8_t>(type_type), 0);
}
void add_type(flatbuffers::Offset<void> type) {
fbb_.AddOffset(SparseTensor::VT_TYPE, type);
}
void add_shape(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TensorDim>>> shape) {
fbb_.AddOffset(SparseTensor::VT_SHAPE, shape);
}
void add_non_zero_length(int64_t non_zero_length) {
fbb_.AddElement<int64_t>(SparseTensor::VT_NON_ZERO_LENGTH, non_zero_length, 0);
}
void add_sparseIndex_type(SparseTensorIndex sparseIndex_type) {
fbb_.AddElement<uint8_t>(SparseTensor::VT_SPARSEINDEX_TYPE, static_cast<uint8_t>(sparseIndex_type), 0);
}
void add_sparseIndex(flatbuffers::Offset<void> sparseIndex) {
fbb_.AddOffset(SparseTensor::VT_SPARSEINDEX, sparseIndex);
}
void add_data(const Buffer *data) {
fbb_.AddStruct(SparseTensor::VT_DATA, data);
}
explicit SparseTensorBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
SparseTensorBuilder &operator=(const SparseTensorBuilder &);
flatbuffers::Offset<SparseTensor> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<SparseTensor>(end);
return o;
}
};
inline flatbuffers::Offset<SparseTensor> CreateSparseTensor(
flatbuffers::FlatBufferBuilder &_fbb,
Type type_type = Type_NONE,
flatbuffers::Offset<void> type = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TensorDim>>> shape = 0,
int64_t non_zero_length = 0,
SparseTensorIndex sparseIndex_type = SparseTensorIndex_NONE,
flatbuffers::Offset<void> sparseIndex = 0,
const Buffer *data = 0) {
SparseTensorBuilder builder_(_fbb);
builder_.add_non_zero_length(non_zero_length);
builder_.add_data(data);
builder_.add_sparseIndex(sparseIndex);
builder_.add_shape(shape);
builder_.add_type(type);
builder_.add_sparseIndex_type(sparseIndex_type);
builder_.add_type_type(type_type);
return builder_.Finish();
}
inline flatbuffers::Offset<SparseTensor> CreateSparseTensorDirect(
flatbuffers::FlatBufferBuilder &_fbb,
Type type_type = Type_NONE,
flatbuffers::Offset<void> type = 0,
const std::vector<flatbuffers::Offset<TensorDim>> *shape = nullptr,
int64_t non_zero_length = 0,
SparseTensorIndex sparseIndex_type = SparseTensorIndex_NONE,
flatbuffers::Offset<void> sparseIndex = 0,
const Buffer *data = 0) {
auto shape__ = shape ? _fbb.CreateVector<flatbuffers::Offset<TensorDim>>(*shape) : 0;
return org::apache::arrow::flatbuf::CreateSparseTensor(
_fbb,
type_type,
type,
shape__,
non_zero_length,
sparseIndex_type,
sparseIndex,
data);
}
inline bool VerifySparseTensorIndex(flatbuffers::Verifier &verifier, const void *obj, SparseTensorIndex type) {
switch (type) {
case SparseTensorIndex_NONE: {
return true;
}
case SparseTensorIndex_SparseTensorIndexCOO: {
auto ptr = reinterpret_cast<const SparseTensorIndexCOO *>(obj);
return verifier.VerifyTable(ptr);
}
case SparseTensorIndex_SparseMatrixIndexCSR: {
auto ptr = reinterpret_cast<const SparseMatrixIndexCSR *>(obj);
return verifier.VerifyTable(ptr);
}
default: return false;
}
}
inline bool VerifySparseTensorIndexVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types) {
if (!values || !types) return !values && !types;
if (values->size() != types->size()) return false;
for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {
if (!VerifySparseTensorIndex(
verifier, values->Get(i), types->GetEnum<SparseTensorIndex>(i))) {
return false;
}
}
return true;
}
inline const org::apache::arrow::flatbuf::SparseTensor *GetSparseTensor(const void *buf) {
return flatbuffers::GetRoot<org::apache::arrow::flatbuf::SparseTensor>(buf);
}
inline const org::apache::arrow::flatbuf::SparseTensor *GetSizePrefixedSparseTensor(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<org::apache::arrow::flatbuf::SparseTensor>(buf);
}
inline bool VerifySparseTensorBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<org::apache::arrow::flatbuf::SparseTensor>(nullptr);
}
inline bool VerifySizePrefixedSparseTensorBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<org::apache::arrow::flatbuf::SparseTensor>(nullptr);
}
inline void FinishSparseTensorBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::SparseTensor> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedSparseTensorBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::SparseTensor> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace flatbuf
} // namespace arrow
} // namespace apache
} // namespace org
#endif // FLATBUFFERS_GENERATED_SPARSETENSOR_ORG_APACHE_ARROW_FLATBUF_H_

377
contrib/arrow-cmake/cpp/src/arrow/ipc/Tensor_generated.h
View File

@ -1,377 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_TENSOR_ORG_APACHE_ARROW_FLATBUF_H_
#define FLATBUFFERS_GENERATED_TENSOR_ORG_APACHE_ARROW_FLATBUF_H_
#include "flatbuffers/flatbuffers.h"
#include "Schema_generated.h"
namespace org {
namespace apache {
namespace arrow {
namespace flatbuf {
struct TensorDim;
struct Tensor;
/// ----------------------------------------------------------------------
/// Data structures for dense tensors
/// Shape data for a single axis in a tensor
struct TensorDim FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_SIZE = 4,
VT_NAME = 6
};
/// Length of dimension
int64_t size() const {
return GetField<int64_t>(VT_SIZE, 0);
}
/// Name of the dimension, optional
const flatbuffers::String *name() const {
return GetPointer<const flatbuffers::String *>(VT_NAME);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int64_t>(verifier, VT_SIZE) &&
VerifyOffset(verifier, VT_NAME) &&
verifier.VerifyString(name()) &&
verifier.EndTable();
}
};
struct TensorDimBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_size(int64_t size) {
fbb_.AddElement<int64_t>(TensorDim::VT_SIZE, size, 0);
}
void add_name(flatbuffers::Offset<flatbuffers::String> name) {
fbb_.AddOffset(TensorDim::VT_NAME, name);
}
explicit TensorDimBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
TensorDimBuilder &operator=(const TensorDimBuilder &);
flatbuffers::Offset<TensorDim> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TensorDim>(end);
return o;
}
};
inline flatbuffers::Offset<TensorDim> CreateTensorDim(
flatbuffers::FlatBufferBuilder &_fbb,
int64_t size = 0,
flatbuffers::Offset<flatbuffers::String> name = 0) {
TensorDimBuilder builder_(_fbb);
builder_.add_size(size);
builder_.add_name(name);
return builder_.Finish();
}
inline flatbuffers::Offset<TensorDim> CreateTensorDimDirect(
flatbuffers::FlatBufferBuilder &_fbb,
int64_t size = 0,
const char *name = nullptr) {
auto name__ = name ? _fbb.CreateString(name) : 0;
return org::apache::arrow::flatbuf::CreateTensorDim(
_fbb,
size,
name__);
}
struct Tensor FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_TYPE_TYPE = 4,
VT_TYPE = 6,
VT_SHAPE = 8,
VT_STRIDES = 10,
VT_DATA = 12
};
Type type_type() const {
return static_cast<Type>(GetField<uint8_t>(VT_TYPE_TYPE, 0));
}
/// The type of data contained in a value cell. Currently only fixed-width
/// value types are supported, no strings or nested types
const void *type() const {
return GetPointer<const void *>(VT_TYPE);
}
template<typename T> const T *type_as() const;
const Null *type_as_Null() const {
return type_type() == Type_Null ? static_cast<const Null *>(type()) : nullptr;
}
const Int *type_as_Int() const {
return type_type() == Type_Int ? static_cast<const Int *>(type()) : nullptr;
}
const FloatingPoint *type_as_FloatingPoint() const {
return type_type() == Type_FloatingPoint ? static_cast<const FloatingPoint *>(type()) : nullptr;
}
const Binary *type_as_Binary() const {
return type_type() == Type_Binary ? static_cast<const Binary *>(type()) : nullptr;
}
const Utf8 *type_as_Utf8() const {
return type_type() == Type_Utf8 ? static_cast<const Utf8 *>(type()) : nullptr;
}
const Bool *type_as_Bool() const {
return type_type() == Type_Bool ? static_cast<const Bool *>(type()) : nullptr;
}
const Decimal *type_as_Decimal() const {
return type_type() == Type_Decimal ? static_cast<const Decimal *>(type()) : nullptr;
}
const Date *type_as_Date() const {
return type_type() == Type_Date ? static_cast<const Date *>(type()) : nullptr;
}
const Time *type_as_Time() const {
return type_type() == Type_Time ? static_cast<const Time *>(type()) : nullptr;
}
const Timestamp *type_as_Timestamp() const {
return type_type() == Type_Timestamp ? static_cast<const Timestamp *>(type()) : nullptr;
}
const Interval *type_as_Interval() const {
return type_type() == Type_Interval ? static_cast<const Interval *>(type()) : nullptr;
}
const List *type_as_List() const {
return type_type() == Type_List ? static_cast<const List *>(type()) : nullptr;
}
const Struct_ *type_as_Struct_() const {
return type_type() == Type_Struct_ ? static_cast<const Struct_ *>(type()) : nullptr;
}
const Union *type_as_Union() const {
return type_type() == Type_Union ? static_cast<const Union *>(type()) : nullptr;
}
const FixedSizeBinary *type_as_FixedSizeBinary() const {
return type_type() == Type_FixedSizeBinary ? static_cast<const FixedSizeBinary *>(type()) : nullptr;
}
const FixedSizeList *type_as_FixedSizeList() const {
return type_type() == Type_FixedSizeList ? static_cast<const FixedSizeList *>(type()) : nullptr;
}
const Map *type_as_Map() const {
return type_type() == Type_Map ? static_cast<const Map *>(type()) : nullptr;
}
const Duration *type_as_Duration() const {
return type_type() == Type_Duration ? static_cast<const Duration *>(type()) : nullptr;
}
const LargeBinary *type_as_LargeBinary() const {
return type_type() == Type_LargeBinary ? static_cast<const LargeBinary *>(type()) : nullptr;
}
const LargeUtf8 *type_as_LargeUtf8() const {
return type_type() == Type_LargeUtf8 ? static_cast<const LargeUtf8 *>(type()) : nullptr;
}
const LargeList *type_as_LargeList() const {
return type_type() == Type_LargeList ? static_cast<const LargeList *>(type()) : nullptr;
}
/// The dimensions of the tensor, optionally named
const flatbuffers::Vector<flatbuffers::Offset<TensorDim>> *shape() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<TensorDim>> *>(VT_SHAPE);
}
/// Non-negative byte offsets to advance one value cell along each dimension
const flatbuffers::Vector<int64_t> *strides() const {
return GetPointer<const flatbuffers::Vector<int64_t> *>(VT_STRIDES);
}
/// The location and size of the tensor's data
const Buffer *data() const {
return GetStruct<const Buffer *>(VT_DATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<uint8_t>(verifier, VT_TYPE_TYPE) &&
VerifyOffset(verifier, VT_TYPE) &&
VerifyType(verifier, type(), type_type()) &&
VerifyOffset(verifier, VT_SHAPE) &&
verifier.VerifyVector(shape()) &&
verifier.VerifyVectorOfTables(shape()) &&
VerifyOffset(verifier, VT_STRIDES) &&
verifier.VerifyVector(strides()) &&
VerifyField<Buffer>(verifier, VT_DATA) &&
verifier.EndTable();
}
};
template<> inline const Null *Tensor::type_as<Null>() const {
return type_as_Null();
}
template<> inline const Int *Tensor::type_as<Int>() const {
return type_as_Int();
}
template<> inline const FloatingPoint *Tensor::type_as<FloatingPoint>() const {
return type_as_FloatingPoint();
}
template<> inline const Binary *Tensor::type_as<Binary>() const {
return type_as_Binary();
}
template<> inline const Utf8 *Tensor::type_as<Utf8>() const {
return type_as_Utf8();
}
template<> inline const Bool *Tensor::type_as<Bool>() const {
return type_as_Bool();
}
template<> inline const Decimal *Tensor::type_as<Decimal>() const {
return type_as_Decimal();
}
template<> inline const Date *Tensor::type_as<Date>() const {
return type_as_Date();
}
template<> inline const Time *Tensor::type_as<Time>() const {
return type_as_Time();
}
template<> inline const Timestamp *Tensor::type_as<Timestamp>() const {
return type_as_Timestamp();
}
template<> inline const Interval *Tensor::type_as<Interval>() const {
return type_as_Interval();
}
template<> inline const List *Tensor::type_as<List>() const {
return type_as_List();
}
template<> inline const Struct_ *Tensor::type_as<Struct_>() const {
return type_as_Struct_();
}
template<> inline const Union *Tensor::type_as<Union>() const {
return type_as_Union();
}
template<> inline const FixedSizeBinary *Tensor::type_as<FixedSizeBinary>() const {
return type_as_FixedSizeBinary();
}
template<> inline const FixedSizeList *Tensor::type_as<FixedSizeList>() const {
return type_as_FixedSizeList();
}
template<> inline const Map *Tensor::type_as<Map>() const {
return type_as_Map();
}
template<> inline const Duration *Tensor::type_as<Duration>() const {
return type_as_Duration();
}
template<> inline const LargeBinary *Tensor::type_as<LargeBinary>() const {
return type_as_LargeBinary();
}
template<> inline const LargeUtf8 *Tensor::type_as<LargeUtf8>() const {
return type_as_LargeUtf8();
}
template<> inline const LargeList *Tensor::type_as<LargeList>() const {
return type_as_LargeList();
}
struct TensorBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_type_type(Type type_type) {
fbb_.AddElement<uint8_t>(Tensor::VT_TYPE_TYPE, static_cast<uint8_t>(type_type), 0);
}
void add_type(flatbuffers::Offset<void> type) {
fbb_.AddOffset(Tensor::VT_TYPE, type);
}
void add_shape(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TensorDim>>> shape) {
fbb_.AddOffset(Tensor::VT_SHAPE, shape);
}
void add_strides(flatbuffers::Offset<flatbuffers::Vector<int64_t>> strides) {
fbb_.AddOffset(Tensor::VT_STRIDES, strides);
}
void add_data(const Buffer *data) {
fbb_.AddStruct(Tensor::VT_DATA, data);
}
explicit TensorBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
TensorBuilder &operator=(const TensorBuilder &);
flatbuffers::Offset<Tensor> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Tensor>(end);
return o;
}
};
inline flatbuffers::Offset<Tensor> CreateTensor(
flatbuffers::FlatBufferBuilder &_fbb,
Type type_type = Type_NONE,
flatbuffers::Offset<void> type = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<TensorDim>>> shape = 0,
flatbuffers::Offset<flatbuffers::Vector<int64_t>> strides = 0,
const Buffer *data = 0) {
TensorBuilder builder_(_fbb);
builder_.add_data(data);
builder_.add_strides(strides);
builder_.add_shape(shape);
builder_.add_type(type);
builder_.add_type_type(type_type);
return builder_.Finish();
}
inline flatbuffers::Offset<Tensor> CreateTensorDirect(
flatbuffers::FlatBufferBuilder &_fbb,
Type type_type = Type_NONE,
flatbuffers::Offset<void> type = 0,
const std::vector<flatbuffers::Offset<TensorDim>> *shape = nullptr,
const std::vector<int64_t> *strides = nullptr,
const Buffer *data = 0) {
auto shape__ = shape ? _fbb.CreateVector<flatbuffers::Offset<TensorDim>>(*shape) : 0;
auto strides__ = strides ? _fbb.CreateVector<int64_t>(*strides) : 0;
return org::apache::arrow::flatbuf::CreateTensor(
_fbb,
type_type,
type,
shape__,
strides__,
data);
}
inline const org::apache::arrow::flatbuf::Tensor *GetTensor(const void *buf) {
return flatbuffers::GetRoot<org::apache::arrow::flatbuf::Tensor>(buf);
}
inline const org::apache::arrow::flatbuf::Tensor *GetSizePrefixedTensor(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<org::apache::arrow::flatbuf::Tensor>(buf);
}
inline bool VerifyTensorBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<org::apache::arrow::flatbuf::Tensor>(nullptr);
}
inline bool VerifySizePrefixedTensorBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<org::apache::arrow::flatbuf::Tensor>(nullptr);
}
inline void FinishTensorBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Tensor> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedTensorBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<org::apache::arrow::flatbuf::Tensor> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace flatbuf
} // namespace arrow
} // namespace apache
} // namespace org
#endif // FLATBUFFERS_GENERATED_TENSOR_ORG_APACHE_ARROW_FLATBUF_H_

839
contrib/arrow-cmake/cpp/src/arrow/ipc/feather_generated.h
View File

@ -1,839 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_FEATHER_ARROW_IPC_FEATHER_FBS_H_
#define FLATBUFFERS_GENERATED_FEATHER_ARROW_IPC_FEATHER_FBS_H_
#include "flatbuffers/flatbuffers.h"
namespace arrow {
namespace ipc {
namespace feather {
namespace fbs {
struct PrimitiveArray;
struct CategoryMetadata;
struct TimestampMetadata;
struct DateMetadata;
struct TimeMetadata;
struct Column;
struct CTable;
/// Feather is an experimental serialization format implemented using
/// techniques from Apache Arrow. It was created as a proof-of-concept of an
/// interoperable file format for storing data frames originating in Python or
/// R. It enabled the developers to sidestep some of the open design questions
/// in Arrow from early 2016 and instead create something simple and useful for
/// the intended use cases.
enum Type {
Type_BOOL = 0,
Type_INT8 = 1,
Type_INT16 = 2,
Type_INT32 = 3,
Type_INT64 = 4,
Type_UINT8 = 5,
Type_UINT16 = 6,
Type_UINT32 = 7,
Type_UINT64 = 8,
Type_FLOAT = 9,
Type_DOUBLE = 10,
Type_UTF8 = 11,
Type_BINARY = 12,
Type_CATEGORY = 13,
Type_TIMESTAMP = 14,
Type_DATE = 15,
Type_TIME = 16,
Type_LARGE_UTF8 = 17,
Type_LARGE_BINARY = 18,
Type_MIN = Type_BOOL,
Type_MAX = Type_LARGE_BINARY
};
inline const Type (&EnumValuesType())[19] {
static const Type values[] = {
Type_BOOL,
Type_INT8,
Type_INT16,
Type_INT32,
Type_INT64,
Type_UINT8,
Type_UINT16,
Type_UINT32,
Type_UINT64,
Type_FLOAT,
Type_DOUBLE,
Type_UTF8,
Type_BINARY,
Type_CATEGORY,
Type_TIMESTAMP,
Type_DATE,
Type_TIME,
Type_LARGE_UTF8,
Type_LARGE_BINARY
};
return values;
}
inline const char * const *EnumNamesType() {
static const char * const names[] = {
"BOOL",
"INT8",
"INT16",
"INT32",
"INT64",
"UINT8",
"UINT16",
"UINT32",
"UINT64",
"FLOAT",
"DOUBLE",
"UTF8",
"BINARY",
"CATEGORY",
"TIMESTAMP",
"DATE",
"TIME",
"LARGE_UTF8",
"LARGE_BINARY",
nullptr
};
return names;
}
inline const char *EnumNameType(Type e) {
if (e < Type_BOOL || e > Type_LARGE_BINARY) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesType()[index];
}
enum Encoding {
Encoding_PLAIN = 0 /// Data is stored dictionary-encoded
/// dictionary size: <INT32 Dictionary size>
/// dictionary data: <TYPE primitive array>
/// dictionary index: <INT32 primitive array>
///
/// TODO: do we care about storing the index values in a smaller typeclass
,
Encoding_DICTIONARY = 1,
Encoding_MIN = Encoding_PLAIN,
Encoding_MAX = Encoding_DICTIONARY
};
inline const Encoding (&EnumValuesEncoding())[2] {
static const Encoding values[] = {
Encoding_PLAIN,
Encoding_DICTIONARY
};
return values;
}
inline const char * const *EnumNamesEncoding() {
static const char * const names[] = {
"PLAIN",
"DICTIONARY",
nullptr
};
return names;
}
inline const char *EnumNameEncoding(Encoding e) {
if (e < Encoding_PLAIN || e > Encoding_DICTIONARY) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesEncoding()[index];
}
enum TimeUnit {
TimeUnit_SECOND = 0,
TimeUnit_MILLISECOND = 1,
TimeUnit_MICROSECOND = 2,
TimeUnit_NANOSECOND = 3,
TimeUnit_MIN = TimeUnit_SECOND,
TimeUnit_MAX = TimeUnit_NANOSECOND
};
inline const TimeUnit (&EnumValuesTimeUnit())[4] {
static const TimeUnit values[] = {
TimeUnit_SECOND,
TimeUnit_MILLISECOND,
TimeUnit_MICROSECOND,
TimeUnit_NANOSECOND
};
return values;
}
inline const char * const *EnumNamesTimeUnit() {
static const char * const names[] = {
"SECOND",
"MILLISECOND",
"MICROSECOND",
"NANOSECOND",
nullptr
};
return names;
}
inline const char *EnumNameTimeUnit(TimeUnit e) {
if (e < TimeUnit_SECOND || e > TimeUnit_NANOSECOND) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesTimeUnit()[index];
}
enum TypeMetadata {
TypeMetadata_NONE = 0,
TypeMetadata_CategoryMetadata = 1,
TypeMetadata_TimestampMetadata = 2,
TypeMetadata_DateMetadata = 3,
TypeMetadata_TimeMetadata = 4,
TypeMetadata_MIN = TypeMetadata_NONE,
TypeMetadata_MAX = TypeMetadata_TimeMetadata
};
inline const TypeMetadata (&EnumValuesTypeMetadata())[5] {
static const TypeMetadata values[] = {
TypeMetadata_NONE,
TypeMetadata_CategoryMetadata,
TypeMetadata_TimestampMetadata,
TypeMetadata_DateMetadata,
TypeMetadata_TimeMetadata
};
return values;
}
inline const char * const *EnumNamesTypeMetadata() {
static const char * const names[] = {
"NONE",
"CategoryMetadata",
"TimestampMetadata",
"DateMetadata",
"TimeMetadata",
nullptr
};
return names;
}
inline const char *EnumNameTypeMetadata(TypeMetadata e) {
if (e < TypeMetadata_NONE || e > TypeMetadata_TimeMetadata) return "";
const size_t index = static_cast<size_t>(e);
return EnumNamesTypeMetadata()[index];
}
template<typename T> struct TypeMetadataTraits {
static const TypeMetadata enum_value = TypeMetadata_NONE;
};
template<> struct TypeMetadataTraits<CategoryMetadata> {
static const TypeMetadata enum_value = TypeMetadata_CategoryMetadata;
};
template<> struct TypeMetadataTraits<TimestampMetadata> {
static const TypeMetadata enum_value = TypeMetadata_TimestampMetadata;
};
template<> struct TypeMetadataTraits<DateMetadata> {
static const TypeMetadata enum_value = TypeMetadata_DateMetadata;
};
template<> struct TypeMetadataTraits<TimeMetadata> {
static const TypeMetadata enum_value = TypeMetadata_TimeMetadata;
};
bool VerifyTypeMetadata(flatbuffers::Verifier &verifier, const void *obj, TypeMetadata type);
bool VerifyTypeMetadataVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);
struct PrimitiveArray FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_TYPE = 4,
VT_ENCODING = 6,
VT_OFFSET = 8,
VT_LENGTH = 10,
VT_NULL_COUNT = 12,
VT_TOTAL_BYTES = 14
};
Type type() const {
return static_cast<Type>(GetField<int8_t>(VT_TYPE, 0));
}
Encoding encoding() const {
return static_cast<Encoding>(GetField<int8_t>(VT_ENCODING, 0));
}
/// Relative memory offset of the start of the array data excluding the size
/// of the metadata
int64_t offset() const {
return GetField<int64_t>(VT_OFFSET, 0);
}
/// The number of logical values in the array
int64_t length() const {
return GetField<int64_t>(VT_LENGTH, 0);
}
/// The number of observed nulls
int64_t null_count() const {
return GetField<int64_t>(VT_NULL_COUNT, 0);
}
/// The total size of the actual data in the file
int64_t total_bytes() const {
return GetField<int64_t>(VT_TOTAL_BYTES, 0);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int8_t>(verifier, VT_TYPE) &&
VerifyField<int8_t>(verifier, VT_ENCODING) &&
VerifyField<int64_t>(verifier, VT_OFFSET) &&
VerifyField<int64_t>(verifier, VT_LENGTH) &&
VerifyField<int64_t>(verifier, VT_NULL_COUNT) &&
VerifyField<int64_t>(verifier, VT_TOTAL_BYTES) &&
verifier.EndTable();
}
};
struct PrimitiveArrayBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_type(Type type) {
fbb_.AddElement<int8_t>(PrimitiveArray::VT_TYPE, static_cast<int8_t>(type), 0);
}
void add_encoding(Encoding encoding) {
fbb_.AddElement<int8_t>(PrimitiveArray::VT_ENCODING, static_cast<int8_t>(encoding), 0);
}
void add_offset(int64_t offset) {
fbb_.AddElement<int64_t>(PrimitiveArray::VT_OFFSET, offset, 0);
}
void add_length(int64_t length) {
fbb_.AddElement<int64_t>(PrimitiveArray::VT_LENGTH, length, 0);
}
void add_null_count(int64_t null_count) {
fbb_.AddElement<int64_t>(PrimitiveArray::VT_NULL_COUNT, null_count, 0);
}
void add_total_bytes(int64_t total_bytes) {
fbb_.AddElement<int64_t>(PrimitiveArray::VT_TOTAL_BYTES, total_bytes, 0);
}
explicit PrimitiveArrayBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
PrimitiveArrayBuilder &operator=(const PrimitiveArrayBuilder &);
flatbuffers::Offset<PrimitiveArray> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<PrimitiveArray>(end);
return o;
}
};
inline flatbuffers::Offset<PrimitiveArray> CreatePrimitiveArray(
flatbuffers::FlatBufferBuilder &_fbb,
Type type = Type_BOOL,
Encoding encoding = Encoding_PLAIN,
int64_t offset = 0,
int64_t length = 0,
int64_t null_count = 0,
int64_t total_bytes = 0) {
PrimitiveArrayBuilder builder_(_fbb);
builder_.add_total_bytes(total_bytes);
builder_.add_null_count(null_count);
builder_.add_length(length);
builder_.add_offset(offset);
builder_.add_encoding(encoding);
builder_.add_type(type);
return builder_.Finish();
}
struct CategoryMetadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_LEVELS = 4,
VT_ORDERED = 6
};
/// The category codes are presumed to be integers that are valid indexes into
/// the levels array
const PrimitiveArray *levels() const {
return GetPointer<const PrimitiveArray *>(VT_LEVELS);
}
bool ordered() const {
return GetField<uint8_t>(VT_ORDERED, 0) != 0;
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_LEVELS) &&
verifier.VerifyTable(levels()) &&
VerifyField<uint8_t>(verifier, VT_ORDERED) &&
verifier.EndTable();
}
};
struct CategoryMetadataBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_levels(flatbuffers::Offset<PrimitiveArray> levels) {
fbb_.AddOffset(CategoryMetadata::VT_LEVELS, levels);
}
void add_ordered(bool ordered) {
fbb_.AddElement<uint8_t>(CategoryMetadata::VT_ORDERED, static_cast<uint8_t>(ordered), 0);
}
explicit CategoryMetadataBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
CategoryMetadataBuilder &operator=(const CategoryMetadataBuilder &);
flatbuffers::Offset<CategoryMetadata> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<CategoryMetadata>(end);
return o;
}
};
inline flatbuffers::Offset<CategoryMetadata> CreateCategoryMetadata(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<PrimitiveArray> levels = 0,
bool ordered = false) {
CategoryMetadataBuilder builder_(_fbb);
builder_.add_levels(levels);
builder_.add_ordered(ordered);
return builder_.Finish();
}
struct TimestampMetadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_UNIT = 4,
VT_TIMEZONE = 6
};
TimeUnit unit() const {
return static_cast<TimeUnit>(GetField<int8_t>(VT_UNIT, 0));
}
/// Timestamp data is assumed to be UTC, but the time zone is stored here for
/// presentation as localized
const flatbuffers::String *timezone() const {
return GetPointer<const flatbuffers::String *>(VT_TIMEZONE);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int8_t>(verifier, VT_UNIT) &&
VerifyOffset(verifier, VT_TIMEZONE) &&
verifier.VerifyString(timezone()) &&
verifier.EndTable();
}
};
struct TimestampMetadataBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_unit(TimeUnit unit) {
fbb_.AddElement<int8_t>(TimestampMetadata::VT_UNIT, static_cast<int8_t>(unit), 0);
}
void add_timezone(flatbuffers::Offset<flatbuffers::String> timezone) {
fbb_.AddOffset(TimestampMetadata::VT_TIMEZONE, timezone);
}
explicit TimestampMetadataBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
TimestampMetadataBuilder &operator=(const TimestampMetadataBuilder &);
flatbuffers::Offset<TimestampMetadata> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TimestampMetadata>(end);
return o;
}
};
inline flatbuffers::Offset<TimestampMetadata> CreateTimestampMetadata(
flatbuffers::FlatBufferBuilder &_fbb,
TimeUnit unit = TimeUnit_SECOND,
flatbuffers::Offset<flatbuffers::String> timezone = 0) {
TimestampMetadataBuilder builder_(_fbb);
builder_.add_timezone(timezone);
builder_.add_unit(unit);
return builder_.Finish();
}
inline flatbuffers::Offset<TimestampMetadata> CreateTimestampMetadataDirect(
flatbuffers::FlatBufferBuilder &_fbb,
TimeUnit unit = TimeUnit_SECOND,
const char *timezone = nullptr) {
auto timezone__ = timezone ? _fbb.CreateString(timezone) : 0;
return arrow::ipc::feather::fbs::CreateTimestampMetadata(
_fbb,
unit,
timezone__);
}
struct DateMetadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
verifier.EndTable();
}
};
struct DateMetadataBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
explicit DateMetadataBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
DateMetadataBuilder &operator=(const DateMetadataBuilder &);
flatbuffers::Offset<DateMetadata> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<DateMetadata>(end);
return o;
}
};
inline flatbuffers::Offset<DateMetadata> CreateDateMetadata(
flatbuffers::FlatBufferBuilder &_fbb) {
DateMetadataBuilder builder_(_fbb);
return builder_.Finish();
}
struct TimeMetadata FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_UNIT = 4
};
TimeUnit unit() const {
return static_cast<TimeUnit>(GetField<int8_t>(VT_UNIT, 0));
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<int8_t>(verifier, VT_UNIT) &&
verifier.EndTable();
}
};
struct TimeMetadataBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_unit(TimeUnit unit) {
fbb_.AddElement<int8_t>(TimeMetadata::VT_UNIT, static_cast<int8_t>(unit), 0);
}
explicit TimeMetadataBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
TimeMetadataBuilder &operator=(const TimeMetadataBuilder &);
flatbuffers::Offset<TimeMetadata> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<TimeMetadata>(end);
return o;
}
};
inline flatbuffers::Offset<TimeMetadata> CreateTimeMetadata(
flatbuffers::FlatBufferBuilder &_fbb,
TimeUnit unit = TimeUnit_SECOND) {
TimeMetadataBuilder builder_(_fbb);
builder_.add_unit(unit);
return builder_.Finish();
}
struct Column FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_NAME = 4,
VT_VALUES = 6,
VT_METADATA_TYPE = 8,
VT_METADATA = 10,
VT_USER_METADATA = 12
};
const flatbuffers::String *name() const {
return GetPointer<const flatbuffers::String *>(VT_NAME);
}
const PrimitiveArray *values() const {
return GetPointer<const PrimitiveArray *>(VT_VALUES);
}
TypeMetadata metadata_type() const {
return static_cast<TypeMetadata>(GetField<uint8_t>(VT_METADATA_TYPE, 0));
}
const void *metadata() const {
return GetPointer<const void *>(VT_METADATA);
}
template<typename T> const T *metadata_as() const;
const CategoryMetadata *metadata_as_CategoryMetadata() const {
return metadata_type() == TypeMetadata_CategoryMetadata ? static_cast<const CategoryMetadata *>(metadata()) : nullptr;
}
const TimestampMetadata *metadata_as_TimestampMetadata() const {
return metadata_type() == TypeMetadata_TimestampMetadata ? static_cast<const TimestampMetadata *>(metadata()) : nullptr;
}
const DateMetadata *metadata_as_DateMetadata() const {
return metadata_type() == TypeMetadata_DateMetadata ? static_cast<const DateMetadata *>(metadata()) : nullptr;
}
const TimeMetadata *metadata_as_TimeMetadata() const {
return metadata_type() == TypeMetadata_TimeMetadata ? static_cast<const TimeMetadata *>(metadata()) : nullptr;
}
/// This should (probably) be JSON
const flatbuffers::String *user_metadata() const {
return GetPointer<const flatbuffers::String *>(VT_USER_METADATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_NAME) &&
verifier.VerifyString(name()) &&
VerifyOffset(verifier, VT_VALUES) &&
verifier.VerifyTable(values()) &&
VerifyField<uint8_t>(verifier, VT_METADATA_TYPE) &&
VerifyOffset(verifier, VT_METADATA) &&
VerifyTypeMetadata(verifier, metadata(), metadata_type()) &&
VerifyOffset(verifier, VT_USER_METADATA) &&
verifier.VerifyString(user_metadata()) &&
verifier.EndTable();
}
};
template<> inline const CategoryMetadata *Column::metadata_as<CategoryMetadata>() const {
return metadata_as_CategoryMetadata();
}
template<> inline const TimestampMetadata *Column::metadata_as<TimestampMetadata>() const {
return metadata_as_TimestampMetadata();
}
template<> inline const DateMetadata *Column::metadata_as<DateMetadata>() const {
return metadata_as_DateMetadata();
}
template<> inline const TimeMetadata *Column::metadata_as<TimeMetadata>() const {
return metadata_as_TimeMetadata();
}
struct ColumnBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_name(flatbuffers::Offset<flatbuffers::String> name) {
fbb_.AddOffset(Column::VT_NAME, name);
}
void add_values(flatbuffers::Offset<PrimitiveArray> values) {
fbb_.AddOffset(Column::VT_VALUES, values);
}
void add_metadata_type(TypeMetadata metadata_type) {
fbb_.AddElement<uint8_t>(Column::VT_METADATA_TYPE, static_cast<uint8_t>(metadata_type), 0);
}
void add_metadata(flatbuffers::Offset<void> metadata) {
fbb_.AddOffset(Column::VT_METADATA, metadata);
}
void add_user_metadata(flatbuffers::Offset<flatbuffers::String> user_metadata) {
fbb_.AddOffset(Column::VT_USER_METADATA, user_metadata);
}
explicit ColumnBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
ColumnBuilder &operator=(const ColumnBuilder &);
flatbuffers::Offset<Column> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Column>(end);
return o;
}
};
inline flatbuffers::Offset<Column> CreateColumn(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::String> name = 0,
flatbuffers::Offset<PrimitiveArray> values = 0,
TypeMetadata metadata_type = TypeMetadata_NONE,
flatbuffers::Offset<void> metadata = 0,
flatbuffers::Offset<flatbuffers::String> user_metadata = 0) {
ColumnBuilder builder_(_fbb);
builder_.add_user_metadata(user_metadata);
builder_.add_metadata(metadata);
builder_.add_values(values);
builder_.add_name(name);
builder_.add_metadata_type(metadata_type);
return builder_.Finish();
}
inline flatbuffers::Offset<Column> CreateColumnDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const char *name = nullptr,
flatbuffers::Offset<PrimitiveArray> values = 0,
TypeMetadata metadata_type = TypeMetadata_NONE,
flatbuffers::Offset<void> metadata = 0,
const char *user_metadata = nullptr) {
auto name__ = name ? _fbb.CreateString(name) : 0;
auto user_metadata__ = user_metadata ? _fbb.CreateString(user_metadata) : 0;
return arrow::ipc::feather::fbs::CreateColumn(
_fbb,
name__,
values,
metadata_type,
metadata,
user_metadata__);
}
struct CTable FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_DESCRIPTION = 4,
VT_NUM_ROWS = 6,
VT_COLUMNS = 8,
VT_VERSION = 10,
VT_METADATA = 12
};
/// Some text (or a name) metadata about what the file is, optional
const flatbuffers::String *description() const {
return GetPointer<const flatbuffers::String *>(VT_DESCRIPTION);
}
int64_t num_rows() const {
return GetField<int64_t>(VT_NUM_ROWS, 0);
}
const flatbuffers::Vector<flatbuffers::Offset<Column>> *columns() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<Column>> *>(VT_COLUMNS);
}
/// Version number of the Feather format
int32_t version() const {
return GetField<int32_t>(VT_VERSION, 0);
}
/// Table metadata (likely JSON), not yet used
const flatbuffers::String *metadata() const {
return GetPointer<const flatbuffers::String *>(VT_METADATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_DESCRIPTION) &&
verifier.VerifyString(description()) &&
VerifyField<int64_t>(verifier, VT_NUM_ROWS) &&
VerifyOffset(verifier, VT_COLUMNS) &&
verifier.VerifyVector(columns()) &&
verifier.VerifyVectorOfTables(columns()) &&
VerifyField<int32_t>(verifier, VT_VERSION) &&
VerifyOffset(verifier, VT_METADATA) &&
verifier.VerifyString(metadata()) &&
verifier.EndTable();
}
};
struct CTableBuilder {
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_description(flatbuffers::Offset<flatbuffers::String> description) {
fbb_.AddOffset(CTable::VT_DESCRIPTION, description);
}
void add_num_rows(int64_t num_rows) {
fbb_.AddElement<int64_t>(CTable::VT_NUM_ROWS, num_rows, 0);
}
void add_columns(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Column>>> columns) {
fbb_.AddOffset(CTable::VT_COLUMNS, columns);
}
void add_version(int32_t version) {
fbb_.AddElement<int32_t>(CTable::VT_VERSION, version, 0);
}
void add_metadata(flatbuffers::Offset<flatbuffers::String> metadata) {
fbb_.AddOffset(CTable::VT_METADATA, metadata);
}
explicit CTableBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
CTableBuilder &operator=(const CTableBuilder &);
flatbuffers::Offset<CTable> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<CTable>(end);
return o;
}
};
inline flatbuffers::Offset<CTable> CreateCTable(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::String> description = 0,
int64_t num_rows = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Column>>> columns = 0,
int32_t version = 0,
flatbuffers::Offset<flatbuffers::String> metadata = 0) {
CTableBuilder builder_(_fbb);
builder_.add_num_rows(num_rows);
builder_.add_metadata(metadata);
builder_.add_version(version);
builder_.add_columns(columns);
builder_.add_description(description);
return builder_.Finish();
}
inline flatbuffers::Offset<CTable> CreateCTableDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const char *description = nullptr,
int64_t num_rows = 0,
const std::vector<flatbuffers::Offset<Column>> *columns = nullptr,
int32_t version = 0,
const char *metadata = nullptr) {
auto description__ = description ? _fbb.CreateString(description) : 0;
auto columns__ = columns ? _fbb.CreateVector<flatbuffers::Offset<Column>>(*columns) : 0;
auto metadata__ = metadata ? _fbb.CreateString(metadata) : 0;
return arrow::ipc::feather::fbs::CreateCTable(
_fbb,
description__,
num_rows,
columns__,
version,
metadata__);
}
inline bool VerifyTypeMetadata(flatbuffers::Verifier &verifier, const void *obj, TypeMetadata type) {
switch (type) {
case TypeMetadata_NONE: {
return true;
}
case TypeMetadata_CategoryMetadata: {
auto ptr = reinterpret_cast<const CategoryMetadata *>(obj);
return verifier.VerifyTable(ptr);
}
case TypeMetadata_TimestampMetadata: {
auto ptr = reinterpret_cast<const TimestampMetadata *>(obj);
return verifier.VerifyTable(ptr);
}
case TypeMetadata_DateMetadata: {
auto ptr = reinterpret_cast<const DateMetadata *>(obj);
return verifier.VerifyTable(ptr);
}
case TypeMetadata_TimeMetadata: {
auto ptr = reinterpret_cast<const TimeMetadata *>(obj);
return verifier.VerifyTable(ptr);
}
default: return false;
}
}
inline bool VerifyTypeMetadataVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types) {
if (!values || !types) return !values && !types;
if (values->size() != types->size()) return false;
for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {
if (!VerifyTypeMetadata(
verifier, values->Get(i), types->GetEnum<TypeMetadata>(i))) {
return false;
}
}
return true;
}
inline const arrow::ipc::feather::fbs::CTable *GetCTable(const void *buf) {
return flatbuffers::GetRoot<arrow::ipc::feather::fbs::CTable>(buf);
}
inline const arrow::ipc::feather::fbs::CTable *GetSizePrefixedCTable(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<arrow::ipc::feather::fbs::CTable>(buf);
}
inline bool VerifyCTableBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<arrow::ipc::feather::fbs::CTable>(nullptr);
}
inline bool VerifySizePrefixedCTableBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<arrow::ipc::feather::fbs::CTable>(nullptr);
}
inline void FinishCTableBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<arrow::ipc::feather::fbs::CTable> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedCTableBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<arrow::ipc::feather::fbs::CTable> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace fbs
} // namespace feather
} // namespace ipc
} // namespace arrow
#endif // FLATBUFFERS_GENERATED_FEATHER_ARROW_IPC_FEATHER_FBS_H_

91
contrib/curl-cmake/CMakeLists.txt
View File

@ -65,11 +65,6 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
endif()
endif()
# For debug libs and exes, add "-d" postfix
if(NOT DEFINED CMAKE_DEBUG_POSTFIX)
set(CMAKE_DEBUG_POSTFIX "-d")
endif()
# initialize CURL_LIBS
set(CURL_LIBS "")
@ -115,8 +110,6 @@ if(ENABLE_IPV6 AND NOT WIN32)
endif()
endif()
curl_nroff_check()
# We need ansi c-flags, especially on HP
set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
@ -132,21 +125,21 @@ include(CheckCSourceCompiles)
if(ENABLE_THREADED_RESOLVER)
find_package(Threads REQUIRED)
if(WIN32)
set(USE_THREADS_WIN32 ON)
else()
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
endif()
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
endif()
# Check for all needed libraries
check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL)
check_function_exists(gethostname HAVE_GETHOSTNAME)
# We don't want any plugin loading at runtime. It is harmful.
#check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
# This is unneeded.
#check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
set (NOT_NEED_LIBNSL 1)
set (gethostname HAVE_GETHOSTNAME 1)
# From cmake/find/ssl.cmake
if (OPENSSL_FOUND)
@ -167,10 +160,12 @@ if (OPENSSL_FOUND)
endif()
# Check for idn
check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
# No, we don't need that.
# check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
# Check for symbol dlopen (same as HAVE_LIBDL)
check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
# We don't want any plugin loading at runtime. It is harmful.
# check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
# From /cmake/find/zlib.cmake
if (ZLIB_FOUND)
@ -181,7 +176,7 @@ if (ZLIB_FOUND)
list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
endif()
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON)
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" OFF)
if(ENABLE_UNIX_SOCKETS)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
@ -217,14 +212,14 @@ check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H)
check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H)
check_include_file_concat("alloca.h" HAVE_ALLOCA_H)
check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H)
check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
#check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
check_include_file_concat("assert.h" HAVE_ASSERT_H)
check_include_file_concat("crypto.h" HAVE_CRYPTO_H)
check_include_file_concat("des.h" HAVE_DES_H)
check_include_file_concat("err.h" HAVE_ERR_H)
check_include_file_concat("errno.h" HAVE_ERRNO_H)
check_include_file_concat("fcntl.h" HAVE_FCNTL_H)
check_include_file_concat("idn2.h" HAVE_IDN2_H)
#check_include_file_concat("idn2.h" HAVE_IDN2_H)
check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H)
check_include_file_concat("io.h" HAVE_IO_H)
check_include_file_concat("krb.h" HAVE_KRB_H)
@ -259,7 +254,7 @@ check_include_file_concat("x509.h" HAVE_X509_H)
check_include_file_concat("process.h" HAVE_PROCESS_H)
check_include_file_concat("stddef.h" HAVE_STDDEF_H)
check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
#check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
check_include_file_concat("malloc.h" HAVE_MALLOC_H)
check_include_file_concat("memory.h" HAVE_MEMORY_H)
check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
@ -276,30 +271,11 @@ check_type_size("int" SIZEOF_INT)
check_type_size("__int64" SIZEOF___INT64)
check_type_size("long double" SIZEOF_LONG_DOUBLE)
check_type_size("time_t" SIZEOF_TIME_T)
if(NOT HAVE_SIZEOF_SSIZE_T)
if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T)
set(ssize_t long)
endif()
if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T)
set(ssize_t __int64)
endif()
endif()
# off_t is sized later, after the HAVE_FILE_OFFSET_BITS test
if(HAVE_SIZEOF_LONG_LONG)
set(HAVE_LONGLONG 1)
set(HAVE_LL 1)
endif()
set(HAVE_LONGLONG 1)
set(HAVE_LL 1)
find_file(RANDOM_FILE urandom /dev)
mark_as_advanced(RANDOM_FILE)
# Check for some functions that are used
if(HAVE_LIBWS2_32)
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
elseif(HAVE_LIBSOCKET)
set(CMAKE_REQUIRED_LIBRARIES socket)
endif()
set(RANDOM_FILE /dev/urandom)
check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME)
check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET)
@ -311,18 +287,15 @@ check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R)
check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP)
check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
#check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
#check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
#check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM)
if(NOT HAVE_STRNCMPI)
set(HAVE_STRCMPI)
endif()
check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
#check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR)
check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
#check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR)
check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR)
@ -331,8 +304,8 @@ check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF)
check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP)
check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R)
check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
#check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
#check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
@ -340,17 +313,15 @@ check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
#check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO)
set(HAVE_SIGNAL 1)
endif()
set(HAVE_SIGNAL 1)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL)
check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
#check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R)
check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)

14
dbms/programs/server/users.xml
View File

@ -84,25 +84,17 @@
<!-- Quota for user. -->
<quota>default</quota>
<!-- For testing the table filters -->
<databases>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<!-- Simple expression filter -->
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<!-- Complex expression filter -->
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
<!-- Filter with ALIAS column -->
<filtered_table3>
<filter>c = 1</filter>
</filtered_table3>
</test>
</databases>
</databases> -->
</default>
<!-- Example of user with readonly access. -->

11
dbms/src/Access/AccessControlManager.cpp
View File

@ -3,6 +3,7 @@
#include <Access/MemoryAccessStorage.h>
#include <Access/UsersConfigAccessStorage.h>
#include <Access/QuotaContextFactory.h>
#include <Access/RowPolicyContextFactory.h>
namespace DB
@ -21,7 +22,8 @@ namespace
AccessControlManager::AccessControlManager()
: MultipleAccessStorage(createStorages()),
quota_context_factory(std::make_unique<QuotaContextFactory>(*this))
quota_context_factory(std::make_unique<QuotaContextFactory>(*this)),
row_policy_context_factory(std::make_unique<RowPolicyContextFactory>(*this))
{
}
@ -49,4 +51,11 @@ std::vector<QuotaUsageInfo> AccessControlManager::getQuotaUsageInfo() const
{
return quota_context_factory->getUsageInfo();
}
std::shared_ptr<RowPolicyContext> AccessControlManager::getRowPolicyContext(const String & user_name) const
{
return row_policy_context_factory->createContext(user_name);
}
}

5
dbms/src/Access/AccessControlManager.h
View File

@ -22,6 +22,8 @@ namespace DB
class QuotaContext;
class QuotaContextFactory;
struct QuotaUsageInfo;
class RowPolicyContext;
class RowPolicyContextFactory;
/// Manages access control entities.
@ -38,8 +40,11 @@ public:
std::vector<QuotaUsageInfo> getQuotaUsageInfo() const;
std::shared_ptr<RowPolicyContext> getRowPolicyContext(const String & user_name) const;
private:
std::unique_ptr<QuotaContextFactory> quota_context_factory;
std::unique_ptr<RowPolicyContextFactory> row_policy_context_factory;
};
}

3
dbms/src/Access/IAccessEntity.cpp
View File

@ -1,5 +1,6 @@
#include <Access/IAccessEntity.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <common/demangle.h>
@ -9,6 +10,8 @@ String IAccessEntity::getTypeName(std::type_index type)
{
if (type == typeid(Quota))
return "Quota";
if (type == typeid(RowPolicy))
return "Row policy";
return demangle(type.name());
}

111
dbms/src/Access/RowPolicy.cpp Normal file
View File

@ -0,0 +1,111 @@
#include <Access/RowPolicy.h>
#include <Interpreters/Context.h>
#include <Common/quoteString.h>
#include <boost/range/algorithm/equal.hpp>
namespace DB
{
namespace
{
void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_)
{
full_name_.clear();
full_name_.reserve(database_.length() + table_name_.length() + policy_name_.length() + 6);
full_name_ += backQuoteIfNeed(policy_name_);
full_name_ += " ON ";
if (!database_.empty())
{
full_name_ += backQuoteIfNeed(database_);
full_name_ += '.';
}
full_name_ += backQuoteIfNeed(table_name_);
}
}
String RowPolicy::FullNameParts::getFullName() const
{
String full_name;
generateFullNameImpl(database, table_name, policy_name, full_name);
return full_name;
}
String RowPolicy::FullNameParts::getFullName(const Context & context) const
{
String full_name;
generateFullNameImpl(database.empty() ? context.getCurrentDatabase() : database, table_name, policy_name, full_name);
return full_name;
}
void RowPolicy::setDatabase(const String & database_)
{
database = database_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setTableName(const String & table_name_)
{
table_name = table_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setName(const String & policy_name_)
{
policy_name = policy_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setFullName(const String & database_, const String & table_name_, const String & policy_name_)
{
database = database_;
table_name = table_name_;
policy_name = policy_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
bool RowPolicy::equal(const IAccessEntity & other) const
{
if (!IAccessEntity::equal(other))
return false;
const auto & other_policy = typeid_cast<const RowPolicy &>(other);
return (database == other_policy.database) && (table_name == other_policy.table_name) && (policy_name == other_policy.policy_name)
&& boost::range::equal(conditions, other_policy.conditions) && restrictive == other_policy.restrictive
&& (roles == other_policy.roles) && (all_roles == other_policy.all_roles) && (except_roles == other_policy.except_roles);
}
const char * RowPolicy::conditionIndexToString(ConditionIndex index)
{
switch (index)
{
case SELECT_FILTER: return "SELECT_FILTER";
case INSERT_CHECK: return "INSERT_CHECK";
case UPDATE_FILTER: return "UPDATE_FILTER";
case UPDATE_CHECK: return "UPDATE_CHECK";
case DELETE_FILTER: return "DELETE_FILTER";
}
__builtin_unreachable();
}
const char * RowPolicy::conditionIndexToColumnName(ConditionIndex index)
{
switch (index)
{
case SELECT_FILTER: return "select_filter";
case INSERT_CHECK: return "insert_check";
case UPDATE_FILTER: return "update_filter";
case UPDATE_CHECK: return "update_check";
case DELETE_FILTER: return "delete_filter";
}
__builtin_unreachable();
}
}

81
dbms/src/Access/RowPolicy.h Normal file
View File

@ -0,0 +1,81 @@
#pragma once
#include <Access/IAccessEntity.h>
namespace DB
{
class Context;
/** Represents a row level security policy for a table.
*/
struct RowPolicy : public IAccessEntity
{
void setDatabase(const String & database_);
void setTableName(const String & table_name_);
void setName(const String & policy_name_) override;
void setFullName(const String & database_, const String & table_name_, const String & policy_name_);
String getDatabase() const { return database; }
String getTableName() const { return table_name; }
String getName() const override { return policy_name; }
struct FullNameParts
{
String database;
String table_name;
String policy_name;
String getFullName() const;
String getFullName(const Context & context) const;
};
/// Filter is a SQL conditional expression used to figure out which rows should be visible
/// for user or available for modification. If the expression returns NULL or false for some rows
/// those rows are silently suppressed.
/// Check is a SQL condition expression used to check whether a row can be written into
/// the table. If the expression returns NULL or false an exception is thrown.
/// If a conditional expression here is empty it means no filtering is applied.
enum ConditionIndex
{
SELECT_FILTER,
INSERT_CHECK,
UPDATE_FILTER,
UPDATE_CHECK,
DELETE_FILTER,
};
static constexpr size_t MAX_CONDITION_INDEX = 5;
static const char * conditionIndexToString(ConditionIndex index);
static const char * conditionIndexToColumnName(ConditionIndex index);
String conditions[MAX_CONDITION_INDEX];
/// Sets that the policy is permissive.
/// A row is only accessible if at least one of the permissive policies passes,
/// in addition to all the restrictive policies.
void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); }
bool isPermissive() const { return !isRestrictive(); }
/// Sets that the policy is restrictive.
/// A row is only accessible if at least one of the permissive policies passes,
/// in addition to all the restrictive policies.
void setRestrictive(bool restrictive_ = true) { restrictive = restrictive_; }
bool isRestrictive() const { return restrictive; }
bool equal(const IAccessEntity & other) const override;
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<RowPolicy>(); }
/// Which roles or users should use this quota.
Strings roles;
bool all_roles = false;
Strings except_roles;
private:
String database;
String table_name;
String policy_name;
bool restrictive = false;
};
using RowPolicyPtr = std::shared_ptr<const RowPolicy>;
}

59
dbms/src/Access/RowPolicyContext.cpp Normal file
View File

@ -0,0 +1,59 @@
#include <Access/RowPolicyContext.h>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/algorithm/copy.hpp>
namespace DB
{
size_t RowPolicyContext::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const
{
return std::hash<StringRef>{}(database_and_table_name.first) - std::hash<StringRef>{}(database_and_table_name.second);
}
RowPolicyContext::RowPolicyContext()
: atomic_map_of_mixed_conditions(std::make_shared<MapOfMixedConditions>())
{
}
RowPolicyContext::~RowPolicyContext() = default;
RowPolicyContext::RowPolicyContext(const String & user_name_)
: user_name(user_name_)
{}
ASTPtr RowPolicyContext::getCondition(const String & database, const String & table_name, ConditionIndex index) const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
auto it = map_of_mixed_conditions->find({database, table_name});
if (it == map_of_mixed_conditions->end())
return {};
return it->second.mixed_conditions[index];
}
std::vector<UUID> RowPolicyContext::getCurrentPolicyIDs() const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
std::vector<UUID> policy_ids;
for (const auto & mixed_conditions : *map_of_mixed_conditions | boost::adaptors::map_values)
boost::range::copy(mixed_conditions.policy_ids, std::back_inserter(policy_ids));
return policy_ids;
}
std::vector<UUID> RowPolicyContext::getCurrentPolicyIDs(const String & database, const String & table_name) const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
auto it = map_of_mixed_conditions->find({database, table_name});
if (it == map_of_mixed_conditions->end())
return {};
return it->second.policy_ids;
}
}

66
dbms/src/Access/RowPolicyContext.h Normal file
View File

@ -0,0 +1,66 @@
#pragma once
#include <Access/RowPolicy.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <common/StringRef.h>
#include <memory>
#include <unordered_map>
namespace DB
{
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
/// Provides fast access to row policies' conditions for a specific user and tables.
class RowPolicyContext
{
public:
/// Default constructor makes a row policy usage context which restricts nothing.
RowPolicyContext();
~RowPolicyContext();
using ConditionIndex = RowPolicy::ConditionIndex;
/// Returns prepared filter for a specific table and operations.
/// The function can return nullptr, that means there is no filters applied.
/// The returned filter can be a combination of the filters defined by multiple row policies.
ASTPtr getCondition(const String & database, const String & table_name, ConditionIndex index) const;
/// Returns IDs of all the policies used by the current user.
std::vector<UUID> getCurrentPolicyIDs() const;
/// Returns IDs of the policies used by a concrete table.
std::vector<UUID> getCurrentPolicyIDs(const String & database, const String & table_name) const;
private:
friend class RowPolicyContextFactory;
friend struct ext::shared_ptr_helper<RowPolicyContext>;
RowPolicyContext(const String & user_name_); /// RowPolicyContext should be created by RowPolicyContextFactory.
using DatabaseAndTableName = std::pair<String, String>;
using DatabaseAndTableNameRef = std::pair<StringRef, StringRef>;
struct Hash
{
size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const;
};
static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX;
using ParsedConditions = std::array<ASTPtr, MAX_CONDITION_INDEX>;
struct MixedConditions
{
std::unique_ptr<DatabaseAndTableName> database_and_table_name_keeper;
ParsedConditions mixed_conditions;
std::vector<UUID> policy_ids;
};
using MapOfMixedConditions = std::unordered_map<DatabaseAndTableNameRef, MixedConditions, Hash>;
const String user_name;
std::shared_ptr<const MapOfMixedConditions> atomic_map_of_mixed_conditions; /// Changed atomically, not protected by `mutex`.
};
using RowPolicyContextPtr = std::shared_ptr<RowPolicyContext>;
}

314
dbms/src/Access/RowPolicyContextFactory.cpp Normal file
View File

@ -0,0 +1,314 @@
#include <Access/RowPolicyContextFactory.h>
#include <Access/RowPolicyContext.h>
#include <Access/AccessControlManager.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/parseQuery.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <ext/range.h>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
namespace DB
{
namespace
{
bool tryGetLiteralBool(const IAST & ast, bool & value)
{
try
{
if (const ASTLiteral * literal = ast.as<ASTLiteral>())
{
value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber<bool>(), literal->value);
return true;
}
return false;
}
catch (...)
{
return false;
}
}
ASTPtr applyFunctionAND(ASTs arguments)
{
bool const_arguments = true;
boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool
{
bool b;
if (!tryGetLiteralBool(*argument, b))
return false;
const_arguments &= b;
return true;
});
if (!const_arguments)
return std::make_shared<ASTLiteral>(Field{UInt8(0)});
if (arguments.empty())
return std::make_shared<ASTLiteral>(Field{UInt8(1)});
if (arguments.size() == 1)
return arguments[0];
auto function = std::make_shared<ASTFunction>();
auto exp_list = std::make_shared<ASTExpressionList>();
function->name = "and";
function->arguments = exp_list;
function->children.push_back(exp_list);
exp_list->children = std::move(arguments);
return function;
}
ASTPtr applyFunctionOR(ASTs arguments)
{
bool const_arguments = false;
boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool
{
bool b;
if (!tryGetLiteralBool(*argument, b))
return false;
const_arguments |= b;
return true;
});
if (const_arguments)
return std::make_shared<ASTLiteral>(Field{UInt8(1)});
if (arguments.empty())
return std::make_shared<ASTLiteral>(Field{UInt8(0)});
if (arguments.size() == 1)
return arguments[0];
auto function = std::make_shared<ASTFunction>();
auto exp_list = std::make_shared<ASTExpressionList>();
function->name = "or";
function->arguments = exp_list;
function->children.push_back(exp_list);
exp_list->children = std::move(arguments);
return function;
}
using ConditionIndex = RowPolicy::ConditionIndex;
static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX;
/// Accumulates conditions from multiple row policies and joins them using the AND logical operation.
class ConditionsMixer
{
public:
void add(const ASTPtr & condition, bool is_restrictive)
{
if (!condition)
return;
if (is_restrictive)
restrictions.push_back(condition);
else
permissions.push_back(condition);
}
ASTPtr getResult() &&
{
/// Process permissive conditions.
if (!permissions.empty())
restrictions.push_back(applyFunctionOR(std::move(permissions)));
/// Process restrictive conditions.
if (!restrictions.empty())
return applyFunctionAND(std::move(restrictions));
return nullptr;
}
private:
ASTs permissions;
ASTs restrictions;
};
}
void RowPolicyContextFactory::PolicyInfo::setPolicy(const RowPolicyPtr & policy_)
{
policy = policy_;
boost::range::copy(policy->roles, std::inserter(roles, roles.end()));
all_roles = policy->all_roles;
boost::range::copy(policy->except_roles, std::inserter(except_roles, except_roles.end()));
for (auto index : ext::range_with_static_cast<ConditionIndex>(0, MAX_CONDITION_INDEX))
{
const String & condition = policy->conditions[index];
auto previous_range = std::pair(std::begin(policy->conditions), std::begin(policy->conditions) + index);
auto previous_it = std::find(previous_range.first, previous_range.second, condition);
if (previous_it != previous_range.second)
{
/// The condition is already parsed before.
parsed_conditions[index] = parsed_conditions[previous_it - previous_range.first];
}
else
{
/// Try to parse the condition.
try
{
ParserExpression parser;
parsed_conditions[index] = parseQuery(parser, condition, 0);
}
catch (...)
{
tryLogCurrentException(
&Poco::Logger::get("RowPolicy"),
String("Could not parse the condition ") + RowPolicy::conditionIndexToString(index) + " of row policy "
+ backQuote(policy->getFullName()));
}
}
}
}
bool RowPolicyContextFactory::PolicyInfo::canUseWithContext(const RowPolicyContext & context) const
{
if (roles.count(context.user_name))
return true;
if (all_roles && !except_roles.count(context.user_name))
return true;
return false;
}
RowPolicyContextFactory::RowPolicyContextFactory(const AccessControlManager & access_control_manager_)
: access_control_manager(access_control_manager_)
{
}
RowPolicyContextFactory::~RowPolicyContextFactory() = default;
RowPolicyContextPtr RowPolicyContextFactory::createContext(const String & user_name)
{
std::lock_guard lock{mutex};
ensureAllRowPoliciesRead();
auto context = ext::shared_ptr_helper<RowPolicyContext>::create(user_name);
contexts.push_back(context);
mixConditionsForContext(*context);
return context;
}
void RowPolicyContextFactory::ensureAllRowPoliciesRead()
{
/// `mutex` is already locked.
if (all_policies_read)
return;
all_policies_read = true;
subscription = access_control_manager.subscribeForChanges<RowPolicy>(
[&](const UUID & id, const AccessEntityPtr & entity)
{
if (entity)
rowPolicyAddedOrChanged(id, typeid_cast<RowPolicyPtr>(entity));
else
rowPolicyRemoved(id);
});
for (const UUID & id : access_control_manager.findAll<RowPolicy>())
{
auto quota = access_control_manager.tryRead<RowPolicy>(id);
if (quota)
all_policies.emplace(id, PolicyInfo(quota));
}
}
void RowPolicyContextFactory::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy)
{
std::lock_guard lock{mutex};
auto it = all_policies.find(policy_id);
if (it == all_policies.end())
{
it = all_policies.emplace(policy_id, PolicyInfo(new_policy)).first;
}
else
{
if (it->second.policy == new_policy)
return;
}
auto & info = it->second;
info.setPolicy(new_policy);
mixConditionsForAllContexts();
}
void RowPolicyContextFactory::rowPolicyRemoved(const UUID & policy_id)
{
std::lock_guard lock{mutex};
all_policies.erase(policy_id);
mixConditionsForAllContexts();
}
void RowPolicyContextFactory::mixConditionsForAllContexts()
{
/// `mutex` is already locked.
boost::range::remove_erase_if(
contexts,
[&](const std::weak_ptr<RowPolicyContext> & weak)
{
auto context = weak.lock();
if (!context)
return true; // remove from the `contexts` list.
mixConditionsForContext(*context);
return false; // keep in the `contexts` list.
});
}
void RowPolicyContextFactory::mixConditionsForContext(RowPolicyContext & context)
{
/// `mutex` is already locked.
struct Mixers
{
ConditionsMixer mixers[MAX_CONDITION_INDEX];
std::vector<UUID> policy_ids;
};
using MapOfMixedConditions = RowPolicyContext::MapOfMixedConditions;
using DatabaseAndTableName = RowPolicyContext::DatabaseAndTableName;
using DatabaseAndTableNameRef = RowPolicyContext::DatabaseAndTableNameRef;
using Hash = RowPolicyContext::Hash;
std::unordered_map<DatabaseAndTableName, Mixers, Hash> map_of_mixers;
for (const auto & [policy_id, info] : all_policies)
{
if (info.canUseWithContext(context))
{
const auto & policy = *info.policy;
auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}];
mixers.policy_ids.push_back(policy_id);
for (auto index : ext::range(0, MAX_CONDITION_INDEX))
mixers.mixers[index].add(info.parsed_conditions[index], policy.isRestrictive());
}
}
auto map_of_mixed_conditions = std::make_shared<MapOfMixedConditions>();
for (auto & [database_and_table_name, mixers] : map_of_mixers)
{
auto database_and_table_name_keeper = std::make_unique<DatabaseAndTableName>();
database_and_table_name_keeper->first = database_and_table_name.first;
database_and_table_name_keeper->second = database_and_table_name.second;
auto & mixed_conditions = (*map_of_mixed_conditions)[DatabaseAndTableNameRef{database_and_table_name_keeper->first,
database_and_table_name_keeper->second}];
mixed_conditions.database_and_table_name_keeper = std::move(database_and_table_name_keeper);
mixed_conditions.policy_ids = std::move(mixers.policy_ids);
for (auto index : ext::range(0, MAX_CONDITION_INDEX))
mixed_conditions.mixed_conditions[index] = std::move(mixers.mixers[index]).getResult();
}
std::atomic_store(&context.atomic_map_of_mixed_conditions, std::shared_ptr<const MapOfMixedConditions>{map_of_mixed_conditions});
}
}

54
dbms/src/Access/RowPolicyContextFactory.h Normal file
View File

@ -0,0 +1,54 @@
#pragma once
#include <Access/RowPolicyContext.h>
#include <Access/IAccessStorage.h>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
namespace DB
{
class AccessControlManager;
/// Stores read and parsed row policies.
class RowPolicyContextFactory
{
public:
RowPolicyContextFactory(const AccessControlManager & access_control_manager_);
~RowPolicyContextFactory();
RowPolicyContextPtr createContext(const String & user_name);
private:
using ParsedConditions = RowPolicyContext::ParsedConditions;
struct PolicyInfo
{
PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); }
void setPolicy(const RowPolicyPtr & policy_);
bool canUseWithContext(const RowPolicyContext & context) const;
RowPolicyPtr policy;
std::unordered_set<String> roles;
bool all_roles = false;
std::unordered_set<String> except_roles;
ParsedConditions parsed_conditions;
};
void ensureAllRowPoliciesRead();
void rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy);
void rowPolicyRemoved(const UUID & policy_id);
void mixConditionsForAllContexts();
void mixConditionsForContext(RowPolicyContext & context);
const AccessControlManager & access_control_manager;
std::unordered_map<UUID, PolicyInfo> all_policies;
bool all_policies_read = false;
IAccessStorage::SubscriptionPtr subscription;
std::vector<std::weak_ptr<RowPolicyContext>> contexts;
std::mutex mutex;
};
}

15
dbms/src/Access/SettingsConstraints.cpp
View File

@ -217,9 +217,18 @@ const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(si
void SettingsConstraints::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config)
{
String parent_profile = "profiles." + profile_name + ".profile";
if (config.has(parent_profile))
setProfile(parent_profile, config); // Inheritance of one profile from another.
String elem = "profiles." + profile_name;
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(elem, config_keys);
for (const std::string & key : config_keys)
{
if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile[")) /// Inheritance of profiles from the current one.
setProfile(config.getString(elem + "." + key), config);
else
continue;
}
String path_to_constraints = "profiles." + profile_name + ".constraints";
if (config.has(path_to_constraints))

56
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -1,5 +1,6 @@
#include <Access/UsersConfigAccessStorage.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -15,6 +16,8 @@ namespace
{
if (type == typeid(Quota))
return 'Q';
if (type == typeid(RowPolicy))
return 'P';
return 0;
}
@ -112,6 +115,57 @@ namespace
}
return quotas;
}
std::vector<AccessEntityPtr> parseRowPolicies(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
{
std::vector<AccessEntityPtr> policies;
Poco::Util::AbstractConfiguration::Keys user_names;
config.keys("users", user_names);
for (const String & user_name : user_names)
{
const String databases_config = "users." + user_name + ".databases";
if (config.has(databases_config))
{
Poco::Util::AbstractConfiguration::Keys databases;
config.keys(databases_config, databases);
/// Read tables within databases
for (const String & database : databases)
{
const String database_config = databases_config + "." + database;
Poco::Util::AbstractConfiguration::Keys table_names;
config.keys(database_config, table_names);
/// Read table properties
for (const String & table_name : table_names)
{
const auto filter_config = database_config + "." + table_name + ".filter";
if (config.has(filter_config))
{
try
{
auto policy = std::make_shared<RowPolicy>();
policy->setFullName(database, table_name, user_name);
policy->conditions[RowPolicy::SELECT_FILTER] = config.getString(filter_config);
policy->roles.push_back(user_name);
policies.push_back(policy);
}
catch (...)
{
tryLogCurrentException(
log,
"Could not parse row policy " + backQuote(user_name) + " on table " + backQuoteIfNeed(database) + "."
+ backQuoteIfNeed(table_name));
}
}
}
}
}
}
return policies;
}
}
@ -128,6 +182,8 @@ void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfigur
std::vector<std::pair<UUID, AccessEntityPtr>> all_entities;
for (const auto & entity : parseQuotas(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseRowPolicies(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
memory_storage.setAll(all_entities);
}

119
dbms/src/AggregateFunctions/AggregateFunctionAggThrow.cpp Normal file
View File

@ -0,0 +1,119 @@
#include <memory>
#include <random>
#include <DataTypes/DataTypesNumber.h>
#include <Common/thread_local_rng.h>
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
namespace DB
{
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_THROW;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
namespace
{
struct AggregateFunctionThrowData
{
bool allocated;
AggregateFunctionThrowData() : allocated(true) {}
~AggregateFunctionThrowData()
{
volatile bool * allocated_ptr = &allocated;
if (*allocated_ptr)
*allocated_ptr = false;
else
abort();
}
};
/** Throw on creation with probability specified in parameter.
* It will check correct destruction of the state.
* This is intended to check for exception safety.
*/
class AggregateFunctionThrow final : public IAggregateFunctionDataHelper<AggregateFunctionThrowData, AggregateFunctionThrow>
{
private:
Float64 throw_probability;
public:
AggregateFunctionThrow(const DataTypes & argument_types_, const Array & parameters_, Float64 throw_probability_)
: IAggregateFunctionDataHelper(argument_types_, parameters_), throw_probability(throw_probability_) {}
String getName() const override
{
return "aggThrow";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt8>();
}
void create(AggregateDataPtr place) const override
{
if (std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= throw_probability)
throw Exception("Aggregate function " + getName() + " has thrown exception successfully", ErrorCodes::AGGREGATE_FUNCTION_THROW);
new (place) Data;
}
void destroy(AggregateDataPtr place) const noexcept override
{
data(place).~Data();
}
void add(AggregateDataPtr, const IColumn **, size_t, Arena *) const override
{
}
void merge(AggregateDataPtr, ConstAggregateDataPtr, Arena *) const override
{
}
void serialize(ConstAggregateDataPtr, WriteBuffer & buf) const override
{
char c = 0;
buf.write(c);
}
void deserialize(AggregateDataPtr, ReadBuffer & buf, Arena *) const override
{
char c = 0;
buf.read(c);
}
void insertResultInto(ConstAggregateDataPtr, IColumn & to) const override
{
to.insertDefault();
}
};
}
void registerAggregateFunctionAggThrow(AggregateFunctionFactory & factory)
{
factory.registerFunction("aggThrow", [](const std::string & name, const DataTypes & argument_types, const Array & parameters)
{
Float64 throw_probability = 1.0;
if (parameters.size() == 1)
throw_probability = parameters[0].safeGet<Float64>();
else if (parameters.size() > 1)
throw Exception("Aggregate function " + name + " cannot have more than one parameter", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
return std::make_shared<AggregateFunctionThrow>(argument_types, parameters, throw_probability);
});
}
}

13
dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h
View File

@ -24,11 +24,16 @@ struct AggregateFunctionArgMinMaxData
ResultData result; // the argument at which the minimum/maximum value is reached.
ValueData value; // value for which the minimum/maximum is calculated.
static bool allocatesMemoryInArena()
{
return ResultData::allocatesMemoryInArena() || ValueData::allocatesMemoryInArena();
}
};
/// Returns the first arg value found for the minimum/maximum value. Example: argMax(arg, value).
template <typename Data, bool AllocatesMemoryInArena>
class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data, AllocatesMemoryInArena>>
template <typename Data>
class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data>>
{
private:
const DataTypePtr & type_res;
@ -36,7 +41,7 @@ private:
public:
AggregateFunctionArgMinMax(const DataTypePtr & type_res_, const DataTypePtr & type_val_)
: IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data, AllocatesMemoryInArena>>({type_res_, type_val_}, {}),
: IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data>>({type_res_, type_val_}, {}),
type_res(this->argument_types[0]), type_val(this->argument_types[1])
{
if (!type_val->isComparable())
@ -77,7 +82,7 @@ public:
bool allocatesMemoryInArena() const override
{
return AllocatesMemoryInArena;
return Data::allocatesMemoryInArena();
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override

23
dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
View File

@ -166,6 +166,11 @@ public:
{
return has() && assert_cast<const ColVecType &>(column).getData()[row_num] == value;
}
static bool allocatesMemoryInArena()
{
return false;
}
};
@ -384,6 +389,11 @@ public:
{
return has() && assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) == getStringRef();
}
static bool allocatesMemoryInArena()
{
return true;
}
};
static_assert(
@ -555,6 +565,11 @@ public:
{
return has() && to.value == value;
}
static bool allocatesMemoryInArena()
{
return false;
}
};
@ -675,15 +690,15 @@ struct AggregateFunctionAnyHeavyData : Data
};
template <typename Data, bool use_arena>
class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data, use_arena>>
template <typename Data>
class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data>>
{
private:
DataTypePtr & type;
public:
AggregateFunctionsSingleValue(const DataTypePtr & type_)
: IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data, use_arena>>({type_}, {})
: IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data>>({type_}, {})
, type(this->argument_types[0])
{
if (StringRef(Data::name()) == StringRef("min")
@ -724,7 +739,7 @@ public:
bool allocatesMemoryInArena() const override
{
return use_arena;
return Data::allocatesMemoryInArena();
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override

13
dbms/src/AggregateFunctions/AggregateFunctionResample.h
View File

@ -100,7 +100,18 @@ public:
void create(AggregateDataPtr place) const override
{
for (size_t i = 0; i < total; ++i)
nested_function->create(place + i * size_of_data);
{
try
{
nested_function->create(place + i * size_of_data);
}
catch (...)
{
for (size_t j = 0; j < i; ++j)
nested_function->destroy(place + j * size_of_data);
throw;
}
}
}
void destroy(AggregateDataPtr place) const noexcept override

4
dbms/src/AggregateFunctions/FactoryHelpers.h
View File

@ -23,13 +23,13 @@ inline void assertNoParameters(const std::string & name, const Array & parameter
inline void assertUnary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 1)
throw Exception("Aggregate function " + name + " require single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
throw Exception("Aggregate function " + name + " requires single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
inline void assertBinary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 2)
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
throw Exception("Aggregate function " + name + " requires two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
template<std::size_t maximal_arity>

81
dbms/src/AggregateFunctions/HelpersMinMaxAny.h
View File

@ -13,8 +13,8 @@
namespace DB
{
/// min, max, any, anyLast
template <template <typename, bool> class AggregateFunctionTemplate, template <typename> class Data>
/// min, max, any, anyLast, anyHeavy, etc...
template <template <typename> class AggregateFunctionTemplate, template <typename> class Data>
static IAggregateFunction * createAggregateFunctionSingleValue(const String & name, const DataTypes & argument_types, const Array & parameters)
{
assertNoParameters(name, parameters);
@ -24,26 +24,26 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
WhichDataType which(argument_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>, false>(argument_type);
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>>(argument_type);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Date)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDate::FieldType>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDate::FieldType>>>(argument_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDateTime::FieldType>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDateTime::FieldType>>>(argument_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DateTime64>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DateTime64>>>(argument_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal32>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal32>>>(argument_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal64>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal64>>>(argument_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal128>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal128>>>(argument_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionTemplate<Data<SingleValueDataString>, true>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataString>>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataGeneric>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataGeneric>>(argument_type);
}
@ -53,53 +53,28 @@ static IAggregateFunction * createAggregateFunctionArgMinMaxSecond(const DataTyp
{
WhichDataType which(val_type);
/// If at least one argument is a String, the function can allocate memory in Arena.
bool is_res_type_string = WhichDataType(res_type).isString();
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE && !is_res_type_string) \
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<TYPE>>>, false>(res_type, val_type); \
if (which.idx == TypeIndex::TYPE && is_res_type_string) \
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<TYPE>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::TYPE) \
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<TYPE>>>>(res_type, val_type); \
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (!is_res_type_string)
{
if (which.idx == TypeIndex::Date)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>, false>(res_type, val_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>, true>(res_type, val_type);
}
else
{
if (which.idx == TypeIndex::Date)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>, true>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>, true>(res_type, val_type);
}
if (which.idx == TypeIndex::Date)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>>(res_type, val_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>>(res_type, val_type);
/// But generic implementation doesn't allocate memory in Arena.
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric>>>(res_type, val_type);
}
template <template <typename> class MinMaxData>

2
dbms/src/AggregateFunctions/IAggregateFunction.h
View File

@ -213,7 +213,7 @@ protected:
public:
IAggregateFunctionDataHelper(const DataTypes & argument_types_, const Array & parameters_)
: IAggregateFunctionHelper<Derived>(argument_types_, parameters_) {}
: IAggregateFunctionHelper<Derived>(argument_types_, parameters_) {}
void create(AggregateDataPtr place) const override
{

1
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -42,6 +42,7 @@ void registerAggregateFunctions()
registerAggregateFunctionSimpleLinearRegression(factory);
registerAggregateFunctionMoving(factory);
registerAggregateFunctionCategoricalIV(factory);
registerAggregateFunctionAggThrow(factory);
}
{

1
dbms/src/AggregateFunctions/registerAggregateFunctions.h
View File

@ -34,6 +34,7 @@ void registerAggregateFunctionEntropy(AggregateFunctionFactory &);
void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &);
void registerAggregateFunctionMoving(AggregateFunctionFactory &);
void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &);
void registerAggregateFunctionAggThrow(AggregateFunctionFactory &);
class AggregateFunctionCombinatorFactory;
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);

1
dbms/src/Common/ErrorCodes.cpp
View File

@ -477,6 +477,7 @@ namespace ErrorCodes
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA = 500;
extern const int CANNOT_CREATE_DATABASE = 501;
extern const int CANNOT_SIGQUEUE = 502;
extern const int AGGREGATE_FUNCTION_THROW = 503;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

2
dbms/src/Core/Settings.cpp
View File

@ -37,7 +37,7 @@ void Settings::setProfile(const String & profile_name, const Poco::Util::Abstrac
{
if (key == "constraints")
continue;
if (key == "profile") /// Inheritance of one profile from another.
if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile[")) /// Inheritance of profiles from the current one.
setProfile(config.getString(elem + "." + key), config);
else
set(key, config.getString(elem + "." + key));

9
dbms/src/Core/Settings.h
View File

@ -358,11 +358,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \
M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
M(SettingBool, allow_experimental_multiple_joins_emulation, true, "Emulate multiple joins using subselects", 0) \
M(SettingBool, allow_experimental_cross_to_join_conversion, true, "Convert CROSS JOIN to INNER JOIN if possible", 0) \
M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \
M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.", 0) \
\
M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \
\
@ -390,11 +387,13 @@ struct Settings : public SettingsCollection<Settings>
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
\
M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13", 0) \
M(SettingBool, compile, false, "Obsolete setting, does nothing. Will be removed after 2020-03-13", 0) \
M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13", 0) \
M(SettingUInt64, min_count_to_compile, 0, "Obsolete setting, does nothing. Will be removed after 2020-03-13", 0) \
M(SettingBool, allow_experimental_multiple_joins_emulation, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, allow_experimental_cross_to_join_conversion, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \
DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)
/** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).

123
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -23,12 +23,8 @@ namespace ErrorCodes
}
DatabaseDictionary::DatabaseDictionary(const String & name_)
: name(name_),
log(&Logger::get("DatabaseDictionary(" + name + ")"))
{
}
void DatabaseDictionary::loadStoredObjects(Context &, bool)
: IDatabase(name_),
log(&Logger::get("DatabaseDictionary(" + database_name + ")"))
{
}
@ -69,65 +65,6 @@ bool DatabaseDictionary::isTableExist(
return context.getExternalDictionariesLoader().getCurrentStatus(table_name) != ExternalLoader::Status::NOT_EXIST;
}
bool DatabaseDictionary::isDictionaryExist(
const Context & /*context*/,
const String & /*table_name*/) const
{
return false;
}
DatabaseDictionariesIteratorPtr DatabaseDictionary::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
void DatabaseDictionary::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::removeDictionary(
const Context & /*context*/,
const String & /*table_name*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::attachDictionary(
const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseDictionary::tryGetCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
return nullptr;
}
ASTPtr DatabaseDictionary::getCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
StoragePtr DatabaseDictionary::tryGetTable(
const Context & context,
const String & table_name) const
@ -153,39 +90,6 @@ bool DatabaseDictionary::empty(const Context & context) const
return !context.getExternalDictionariesLoader().hasCurrentlyLoadedObjects();
}
StoragePtr DatabaseDictionary::detachTable(const String & /*table_name*/)
{
throw Exception("DatabaseDictionary: detachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::attachTable(const String & /*table_name*/, const StoragePtr & /*table*/)
{
throw Exception("DatabaseDictionary: attachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::createTable(
const Context &,
const String &,
const StoragePtr &,
const ASTPtr &)
{
throw Exception("DatabaseDictionary: createTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::removeTable(
const Context &,
const String &)
{
throw Exception("DatabaseDictionary: removeTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
time_t DatabaseDictionary::getObjectMetadataModificationTime(
const Context &,
const String &)
{
return static_cast<time_t>(0);
}
ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
const String & table_name, bool throw_on_error) const
{
@ -196,9 +100,11 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
const auto & dictionaries = context.getExternalDictionariesLoader();
auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name)
: dictionaries.tryGetDictionary(table_name);
if (!dictionary)
return {};
auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
buffer << "CREATE TABLE " << backQuoteIfNeed(database_name) << '.' << backQuoteIfNeed(table_name) << " (";
buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
}
@ -215,22 +121,12 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
return ast;
}
ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, true);
}
ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, false);
}
ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const
{
String query;
{
WriteBufferFromString buffer(query);
buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary";
buffer << "CREATE DATABASE " << backQuoteIfNeed(database_name) << " ENGINE = Dictionary";
}
ParserCreateQuery parser;
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
@ -240,9 +136,4 @@ void DatabaseDictionary::shutdown()
{
}
String DatabaseDictionary::getDatabaseName() const
{
return name;
}
}

57
dbms/src/Databases/DatabaseDictionary.h
View File

@ -24,85 +24,36 @@ class DatabaseDictionary : public IDatabase
public:
DatabaseDictionary(const String & name_);
String getDatabaseName() const override;
String getEngineName() const override
{
return "Dictionary";
}
void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) override;
bool isTableExist(
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(const Context & context, const String & table_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool empty(const Context & context) const override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context, const String & dictionary_name, const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(const Context & context, const String & table_name) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
ASTPtr getCreateDictionaryQuery(const Context & context, const String & table_name) const override;
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & table_name) const override;
void attachDictionary(const String & dictionary_name, const Context & context) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
ASTPtr getCreateDatabaseQuery() const override;
void shutdown() override;
protected:
ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const override;
private:
const String name;
mutable std::mutex mutex;
Poco::Logger * log;
Tables listTables(const Context & context, const FilterByNameFunction & filter_by_name);
ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
};
}

159
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -3,7 +3,6 @@
#include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabasesCommon.h>
#include <Interpreters/Context.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
@ -31,11 +30,8 @@ namespace ErrorCodes
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_)
: name(name_)
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(name) + "/")
: DatabaseOnDisk(name_, metadata_path_, "DatabaseLazy (" + name_ + ")")
, expiration_time(expiration_time_)
, log(&Logger::get("DatabaseLazy (" + name + ")"))
{
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -45,7 +41,7 @@ void DatabaseLazy::loadStoredObjects(
Context & context,
bool /* has_force_restore_data_flag */)
{
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [this](const String & file_name)
iterateMetadataFiles(context, [this](const String & file_name)
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
attachTable(table_name, nullptr);
@ -62,75 +58,21 @@ void DatabaseLazy::createTable(
SCOPE_EXIT({ clearExpiredTables(); });
if (!endsWith(table->getName(), "Log"))
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
DatabaseOnDisk::createTable(context, table_name, table, query);
/// DatabaseOnDisk::createTable renames file, so we need to get new metadata_modification_time.
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it != tables_cache.end())
it->second.metadata_modification_time = DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name);
it->second.metadata_modification_time = DatabaseOnDisk::getObjectMetadataModificationTime(table_name);
}
void DatabaseLazy::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::removeTable(
const Context & context,
const String & table_name)
{
SCOPE_EXIT({ clearExpiredTables(); });
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
void DatabaseLazy::removeDictionary(
const Context & /*context*/,
const String & /*table_name*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseLazy::getCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseLazy::tryGetCreateDictionaryQuery(const Context & /*context*/, const String & /*table_name*/) const
{
return nullptr;
}
bool DatabaseLazy::isDictionaryExist(const Context & /*context*/, const String & /*table_name*/) const
{
return false;
}
DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
void DatabaseLazy::attachDictionary(
const String & /*dictionary_name*/,
const Context & /*context*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
DatabaseOnDisk::removeTable(context, table_name);
}
void DatabaseLazy::renameTable(
@ -141,61 +83,34 @@ void DatabaseLazy::renameTable(
TableStructureWriteLockHolder & lock)
{
SCOPE_EXIT({ clearExpiredTables(); });
DatabaseOnDisk::renameTable<DatabaseLazy>(*this, context, table_name, to_database, to_table_name, lock);
DatabaseOnDisk::renameTable(context, table_name, to_database, to_table_name, lock);
}
time_t DatabaseLazy::getObjectMetadataModificationTime(
const Context & /* context */,
const String & table_name)
time_t DatabaseLazy::getObjectMetadataModificationTime(const String & table_name) const
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it != tables_cache.end())
return it->second.metadata_modification_time;
else
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
}
ASTPtr DatabaseLazy::getCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseLazy::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseLazy::getCreateDatabaseQuery(const Context & context) const
{
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
}
void DatabaseLazy::alterTable(
const Context & /* context */,
const String & /* table_name */,
const ColumnsDescription & /* columns */,
const IndicesDescription & /* indices */,
const ConstraintsDescription & /* constraints */,
const ASTModifier & /* storage_modifier */)
const StorageInMemoryMetadata & /* metadata */)
{
SCOPE_EXIT({ clearExpiredTables(); });
clearExpiredTables();
throw Exception("ALTER query is not supported for Lazy database.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::drop(const Context & context)
{
DatabaseOnDisk::drop(*this, context);
}
bool DatabaseLazy::isTableExist(
const Context & /* context */,
const String & table_name) const
{
SCOPE_EXIT({ clearExpiredTables(); });
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
return tables_cache.find(table_name) != tables_cache.end();
}
@ -205,7 +120,7 @@ StoragePtr DatabaseLazy::tryGetTable(
{
SCOPE_EXIT({ clearExpiredTables(); });
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -225,7 +140,7 @@ StoragePtr DatabaseLazy::tryGetTable(
DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
Strings filtered_tables;
for (const auto & [table_name, cached_table] : tables_cache)
{
@ -244,12 +159,12 @@ bool DatabaseLazy::empty(const Context & /* context */) const
void DatabaseLazy::attachTable(const String & table_name, const StoragePtr & table)
{
LOG_DEBUG(log, "Attach table " << backQuote(table_name) << ".");
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
auto [it, inserted] = tables_cache.emplace(std::piecewise_construct,
std::forward_as_tuple(table_name),
std::forward_as_tuple(table, current_time, DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name)));
std::forward_as_tuple(table, current_time, DatabaseOnDisk::getObjectMetadataModificationTime(table_name)));
if (!inserted)
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
@ -261,7 +176,7 @@ StoragePtr DatabaseLazy::detachTable(const String & table_name)
StoragePtr res;
{
LOG_DEBUG(log, "Detach table " << backQuote(table_name) << ".");
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -277,7 +192,7 @@ void DatabaseLazy::shutdown()
{
TablesCache tables_snapshot;
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
tables_snapshot = tables_cache;
}
@ -287,7 +202,7 @@ void DatabaseLazy::shutdown()
kv.second.table->shutdown();
}
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
tables_cache.clear();
}
@ -303,26 +218,6 @@ DatabaseLazy::~DatabaseLazy()
}
}
String DatabaseLazy::getDataPath() const
{
return data_path;
}
String DatabaseLazy::getMetadataPath() const
{
return metadata_path;
}
String DatabaseLazy::getDatabaseName() const
{
return name;
}
String DatabaseLazy::getObjectMetadataPath(const String & table_name) const
{
return DatabaseOnDisk::getObjectMetadataPath(*this, table_name);
}
StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table_name) const
{
SCOPE_EXIT({ clearExpiredTables(); });
@ -333,19 +228,21 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
try
{
String table_name_;
StoragePtr table;
Context context_copy(context); /// some tables can change context, but not LogTables
auto ast = parseCreateQueryFromMetadataFile(table_metadata_path, log);
auto ast = parseQueryFromMetadata(table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast)
std::tie(table_name_, table) = createTableFromAST(
ast->as<const ASTCreateQuery &>(), name, getDataPath(), context_copy, false);
{
auto & ast_create = ast->as<const ASTCreateQuery &>();
String table_data_path_relative = getTableDataPath(ast_create);
table = createTableFromAST(ast_create, database_name, table_data_path_relative, context_copy, false).second;
}
if (!ast || !endsWith(table->getName(), "Log"))
throw Exception("Only *Log tables can be used with Lazy database engine.", ErrorCodes::LOGICAL_ERROR);
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -367,7 +264,7 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
void DatabaseLazy::clearExpiredTables() const
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto time_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
CacheExpirationQueue expired_tables;

66
dbms/src/Databases/DatabaseLazy.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Databases/DatabaseOnDisk.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
@ -15,7 +15,7 @@ class DatabaseLazyIterator;
* Works like DatabaseOrdinary, but stores in memory only cache.
* Can be used only with *Log engines.
*/
class DatabaseLazy : public IDatabase
class DatabaseLazy : public DatabaseOnDisk
{
public:
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_);
@ -32,19 +32,10 @@ public:
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
@ -55,48 +46,14 @@ public:
void alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & engine_modifier) override;
const StorageInMemoryMetadata & metadata) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr getCreateDictionaryQuery(
const Context & context,
const String & dictionary_name) const override;
ASTPtr tryGetCreateDictionaryQuery(
const Context & context,
const String & dictionary_name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
String getDataPath() const override;
String getDatabaseName() const override;
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop(const Context & context) override;
time_t getObjectMetadataModificationTime(const String & table_name) const override;
bool isTableExist(
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(
const Context & context,
const String & table_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
@ -105,16 +62,10 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
void attachDictionary(const String & dictionary_name, const Context & context) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
void shutdown() override;
~DatabaseLazy() override;
@ -146,19 +97,12 @@ private:
using TablesCache = std::unordered_map<String, CachedTable>;
String name;
const String metadata_path;
const String data_path;
const time_t expiration_time;
mutable std::mutex tables_mutex;
/// TODO use DatabaseWithOwnTablesBase::tables
mutable TablesCache tables_cache;
mutable CacheExpirationQueue cache_expiration_queue;
Poco::Logger * log;
StoragePtr loadTable(const Context & context, const String & table_name) const;
void clearExpiredTables() const;

86
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -1,30 +1,16 @@
#include <common/logger_useful.h>
#include <Databases/DatabaseMemory.h>
#include <Databases/DatabasesCommon.h>
#include <Parsers/ASTCreateQuery.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
extern const int UNSUPPORTED_METHOD;
}
DatabaseMemory::DatabaseMemory(String name_)
: DatabaseWithOwnTablesBase(std::move(name_))
, log(&Logger::get("DatabaseMemory(" + name + ")"))
DatabaseMemory::DatabaseMemory(const String & name_)
: DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")")
{}
void DatabaseMemory::loadStoredObjects(
Context & /*context*/,
bool /*has_force_restore_data_flag*/)
{
/// Nothing to load.
}
void DatabaseMemory::createTable(
const Context & /*context*/,
const String & table_name,
@ -34,21 +20,6 @@ void DatabaseMemory::createTable(
attachTable(table_name, table);
}
void DatabaseMemory::attachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no ATTACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::removeTable(
const Context & /*context*/,
const String & table_name)
@ -56,52 +27,13 @@ void DatabaseMemory::removeTable(
detachTable(table_name);
}
void DatabaseMemory::detachDictionary(const String & /*name*/, const Context & /*context*/)
ASTPtr DatabaseMemory::getCreateDatabaseQuery() const
{
throw Exception("There is no DETACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::removeDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/)
{
throw Exception("There is no DROP DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
time_t DatabaseMemory::getObjectMetadataModificationTime(
const Context &, const String &)
{
return static_cast<time_t>(0);
}
ASTPtr DatabaseMemory::getCreateTableQuery(
const Context &,
const String &) const
{
throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
}
ASTPtr DatabaseMemory::getCreateDictionaryQuery(
const Context &,
const String &) const
{
throw Exception("There is no CREATE DICTIONARY query for DatabaseMemory dictionaries", ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
ASTPtr DatabaseMemory::getCreateDatabaseQuery(
const Context &) const
{
throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
}
String DatabaseMemory::getDatabaseName() const
{
return name;
auto create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name;
create_query->set(create_query->storage, std::make_shared<ASTStorage>());
create_query->storage->set(create_query->storage->engine, makeASTFunction(getEngineName()));
return create_query;
}
}

37
dbms/src/Databases/DatabaseMemory.h
View File

@ -17,54 +17,21 @@ namespace DB
class DatabaseMemory : public DatabaseWithOwnTablesBase
{
public:
DatabaseMemory(String name_);
String getDatabaseName() const override;
DatabaseMemory(const String & name_);
String getEngineName() const override { return "Memory"; }
void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void attachDictionary(
const String & name,
const Context & context) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & dictionary_name) override;
void detachDictionary(
const String & name,
const Context & context) override;
time_t getObjectMetadataModificationTime(const Context & context, const String & table_name) override;
ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override;
ASTPtr getCreateDictionaryQuery(const Context & context, const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; }
ASTPtr tryGetCreateDictionaryQuery(const Context &, const String &) const override { return nullptr; }
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
private:
Poco::Logger * log;
ASTPtr getCreateDatabaseQuery() const override;
};
}

23
dbms/src/Databases/DatabaseMySQL.cpp
View File

@ -9,10 +9,8 @@
#include <Formats/MySQLBlockInputStream.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeFixedString.h>
#include <Storages/StorageMySQL.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ParserCreateQuery.h>
@ -63,8 +61,12 @@ static String toQueryStringWithQuote(const std::vector<String> & quote_list)
DatabaseMySQL::DatabaseMySQL(
const Context & global_context_, const String & database_name_, const String & metadata_path_,
const ASTStorage * database_engine_define_, const String & database_name_in_mysql_, mysqlxx::Pool && pool)
: global_context(global_context_), database_name(database_name_), metadata_path(metadata_path_),
database_engine_define(database_engine_define_->clone()), database_name_in_mysql(database_name_in_mysql_), mysql_pool(std::move(pool))
: IDatabase(database_name_)
, global_context(global_context_)
, metadata_path(metadata_path_)
, database_engine_define(database_engine_define_->clone())
, database_name_in_mysql(database_name_in_mysql_)
, mysql_pool(std::move(pool))
{
}
@ -150,19 +152,24 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
return create_table_query;
}
ASTPtr DatabaseMySQL::tryGetCreateTableQuery(const Context &, const String & table_name) const
ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const Context &, const String & table_name, bool throw_on_error) const
{
std::lock_guard<std::mutex> lock(mutex);
fetchTablesIntoLocalCache();
if (local_tables_cache.find(table_name) == local_tables_cache.end())
throw Exception("MySQL table " + database_name_in_mysql + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
{
if (throw_on_error)
throw Exception("MySQL table " + database_name_in_mysql + "." + table_name + " doesn't exist..",
ErrorCodes::UNKNOWN_TABLE);
return nullptr;
}
return getCreateQueryFromStorage(local_tables_cache[table_name].second, database_engine_define);
}
time_t DatabaseMySQL::getObjectMetadataModificationTime(const Context &, const String & table_name)
time_t DatabaseMySQL::getObjectMetadataModificationTime(const String & table_name) const
{
std::lock_guard<std::mutex> lock(mutex);
@ -174,7 +181,7 @@ time_t DatabaseMySQL::getObjectMetadataModificationTime(const Context &, const S
return time_t(local_tables_cache[table_name].first);
}
ASTPtr DatabaseMySQL::getCreateDatabaseQuery(const Context &) const
ASTPtr DatabaseMySQL::getCreateDatabaseQuery() const
{
const auto & create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name;

43
dbms/src/Databases/DatabaseMySQL.h
View File

@ -28,35 +28,17 @@ public:
String getEngineName() const override { return "MySQL"; }
String getDatabaseName() const override { return database_name; }
bool empty(const Context & context) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context &, const FilterByNameFunction & = {}) override
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
ASTPtr getCreateDatabaseQuery() const override;
bool isTableExist(const Context & context, const String & name) const override;
bool isDictionaryExist(const Context &, const String &) const override { return false; }
StoragePtr tryGetTable(const Context & context, const String & name) const override;
ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const override;
ASTPtr getCreateDictionaryQuery(const Context &, const String &) const override
{
throw Exception("MySQL database engine does not support dictionaries.", ErrorCodes::NOT_IMPLEMENTED);
}
ASTPtr tryGetCreateDictionaryQuery(const Context &, const String &) const override { return nullptr; }
time_t getObjectMetadataModificationTime(const Context & context, const String & name) override;
time_t getObjectMetadataModificationTime(const String & name) const override;
void shutdown() override;
@ -74,29 +56,12 @@ public:
void attachTable(const String & table_name, const StoragePtr & storage) override;
void detachDictionary(const String &, const Context &) override
{
throw Exception("MySQL database engine does not support detach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void removeDictionary(const Context &, const String &) override
{
throw Exception("MySQL database engine does not support remove dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void attachDictionary(const String &, const Context &) override
{
throw Exception("MySQL database engine does not support attach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void createDictionary(const Context &, const String &, const ASTPtr &) override
{
throw Exception("MySQL database engine does not support create dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
protected:
ASTPtr getCreateTableQueryImpl(const Context & context, const String & name, bool throw_on_error) const override;
private:
Context global_context;
String database_name;
String metadata_path;
ASTPtr database_engine_define;
String database_name_in_mysql;

416
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -6,9 +6,6 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -19,7 +16,6 @@
#include <Common/escapeForFileName.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include <Poco/DirectoryIterator.h>
@ -31,8 +27,6 @@ static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
namespace ErrorCodes
{
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_FILE_NAME;
extern const int SYNTAX_ERROR;
@ -43,93 +37,10 @@ namespace ErrorCodes
}
namespace detail
{
String getObjectMetadataPath(const String & base_path, const String & table_name)
{
return base_path + (endsWith(base_path, "/") ? "" : "/") + escapeForFileName(table_name) + ".sql";
}
String getDatabaseMetadataPath(const String & base_path)
{
return (endsWith(base_path, "/") ? base_path.substr(0, base_path.size() - 1) : base_path) + ".sql";
}
ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error)
{
String query;
try
{
ReadBufferFromFile in(metadata_path, 4096);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + metadata_path, /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
return ast;
}
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
{
ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database;
}
return ast;
}
}
ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger * log)
{
String definition;
{
char in_buf[METADATA_FILE_BUFFER_SIZE];
ReadBufferFromFile in(filepath, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
readStringUntilEOF(definition, in);
}
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (definition.empty())
{
LOG_ERROR(log, "File " << filepath << " is empty. Removing.");
Poco::File(filepath).remove();
return nullptr;
}
ParserCreateQuery parser_create;
ASTPtr result = parseQuery(parser_create, definition, "in file " + filepath, 0);
return result;
}
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path_relative,
const String & table_data_path_relative,
Context & context,
bool has_force_restore_data_flag)
{
@ -144,7 +55,7 @@ std::pair<String, StoragePtr> createTableFromAST(
return {ast_create_query.table, storage};
}
/// We do not directly use `InterpreterCreateQuery::execute`, because
/// - the database has not been created yet;
/// - the database has not been loaded yet;
/// - the code is simpler, since the query is already brought to a suitable form.
if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
@ -152,7 +63,6 @@ std::pair<String, StoragePtr> createTableFromAST(
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
String table_data_path_relative = database_data_path_relative + escapeForFileName(ast_create_query.table) + '/';
return
{
ast_create_query.table,
@ -202,7 +112,6 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query)
}
void DatabaseOnDisk::createTable(
IDatabase & database,
const Context & context,
const String & table_name,
const StoragePtr & table,
@ -222,14 +131,14 @@ void DatabaseOnDisk::createTable(
/// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (database.isDictionaryExist(context, table_name))
throw Exception("Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
if (isDictionaryExist(context, table_name))
throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (database.isTableExist(context, table_name))
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
if (isTableExist(context, table_name))
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String table_metadata_path = database.getObjectMetadataPath(table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
String table_metadata_tmp_path = table_metadata_path + ".tmp";
String statement;
@ -248,7 +157,7 @@ void DatabaseOnDisk::createTable(
try
{
/// Add a table to the map of known tables.
database.attachTable(table_name, table);
attachTable(table_name, table);
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
@ -261,107 +170,11 @@ void DatabaseOnDisk::createTable(
}
}
void DatabaseOnDisk::createDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
const ASTPtr & query)
void DatabaseOnDisk::removeTable(const Context & /* context */, const String & table_name)
{
const auto & settings = context.getSettingsRef();
StoragePtr res = detachTable(table_name);
/** The code is based on the assumption that all threads share the same order of operations:
* - create the .sql.tmp file;
* - add the dictionary to ExternalDictionariesLoader;
* - load the dictionary in case dictionaries_lazy_load == false;
* - attach the dictionary;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a dictionary with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (database.isDictionaryExist(context, dictionary_name))
throw Exception("Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = database.getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (database.isTableExist(context, dictionary_name))
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp";
String statement = getObjectDefinitionFromCreateQuery(query);
{
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
bool succeeded = false;
SCOPE_EXIT({
if (!succeeded)
Poco::File(dictionary_metadata_tmp_path).remove();
});
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), database.getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
{
/// load() is called here to force loading the dictionary, wait until the loading is finished,
/// and throw an exception if the loading is failed.
external_loader.load(full_name);
}
database.attachDictionary(dictionary_name, context);
SCOPE_EXIT({
if (!succeeded)
database.detachDictionary(dictionary_name, context);
});
/// If it was ATTACH query and file with dictionary metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path);
/// ExternalDictionariesLoader doesn't know we renamed the metadata path.
/// So we have to manually call reloadConfig() here.
external_loader.reloadConfig(database.getDatabaseName(), full_name);
/// Everything's ok.
succeeded = true;
}
void DatabaseOnDisk::removeTable(
IDatabase & database,
const Context & /* context */,
const String & table_name,
Poco::Logger * log)
{
StoragePtr res = database.detachTable(table_name);
String table_metadata_path = database.getObjectMetadataPath(table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
try
{
@ -378,51 +191,64 @@ void DatabaseOnDisk::removeTable(
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
database.attachTable(table_name, res);
attachTable(table_name, res);
throw;
}
}
void DatabaseOnDisk::removeDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
Poco::Logger * /*log*/)
void DatabaseOnDisk::renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
database.detachDictionary(dictionary_name, context);
if (typeid(*this) != typeid(to_database))
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
if (Poco::File(dictionary_metadata_path).exists())
StoragePtr table = tryGetTable(context, table_name);
if (!table)
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name));
if (!ast)
throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
auto & create = ast->as<ASTCreateQuery &>();
create.table = to_table_name;
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
try
{
Poco::File(dictionary_metadata_path).remove();
}
catch (...)
{
/// If remove was not possible for some reason
database.attachDictionary(dictionary_name, context);
throw;
}
table->rename(to_database.getTableDataPath(create),
to_database.getDatabaseName(),
to_table_name, lock);
}
catch (const Exception &)
{
throw;
}
catch (const Poco::Exception & e)
{
/// Better diagnostics.
throw Exception{Exception::CreateFromPoco, e};
}
/// NOTE Non-atomic.
to_database.createTable(context, to_table_name, table, ast);
removeTable(context, table_name);
}
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(
const IDatabase & database,
const Context & context,
const String & table_name,
bool throw_on_error)
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const
{
ASTPtr ast;
auto table_metadata_path = detail::getObjectMetadataPath(database.getMetadataPath(), table_name);
ast = detail::getCreateQueryFromMetadata(table_metadata_path, database.getDatabaseName(), throw_on_error);
auto table_metadata_path = getObjectMetadataPath(table_name);
ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_table = database.tryGetTable(context, table_name) != nullptr;
bool has_table = tryGetTable(context, table_name) != nullptr;
auto msg = has_table
? "There is no CREATE TABLE query for table "
@ -434,61 +260,18 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(
return ast;
}
ASTPtr DatabaseOnDisk::getCreateDictionaryQueryImpl(
const IDatabase & database,
const Context & context,
const String & dictionary_name,
bool throw_on_error)
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
{
ASTPtr ast;
auto dictionary_metadata_path = detail::getObjectMetadataPath(database.getMetadataPath(), dictionary_name);
ast = detail::getCreateQueryFromMetadata(dictionary_metadata_path, database.getDatabaseName(), throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_dictionary = database.isDictionaryExist(context, dictionary_name);
auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
return ast;
}
ASTPtr DatabaseOnDisk::getCreateTableQuery(const IDatabase & database, const Context & context, const String & table_name)
{
return getCreateTableQueryImpl(database, context, table_name, true);
}
ASTPtr DatabaseOnDisk::tryGetCreateTableQuery(const IDatabase & database, const Context & context, const String & table_name)
{
return getCreateTableQueryImpl(database, context, table_name, false);
}
ASTPtr DatabaseOnDisk::getCreateDictionaryQuery(const IDatabase & database, const Context & context, const String & dictionary_name)
{
return getCreateDictionaryQueryImpl(database, context, dictionary_name, true);
}
ASTPtr DatabaseOnDisk::tryGetCreateDictionaryQuery(const IDatabase & database, const Context & context, const String & dictionary_name)
{
return getCreateDictionaryQueryImpl(database, context, dictionary_name, false);
}
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const Context & /*context*/)
{
ASTPtr ast;
auto database_metadata_path = detail::getDatabaseMetadataPath(database.getMetadataPath());
ast = detail::getCreateQueryFromMetadata(database_metadata_path, database.getDatabaseName(), true);
auto metadata_dir_path = getMetadataPath();
auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql";
ast = getCreateQueryFromMetadata(database_metadata_path, true);
if (!ast)
{
/// Handle databases (such as default) for which there are no database.sql files.
String query = "CREATE DATABASE " + backQuoteIfNeed(database.getDatabaseName()) + " ENGINE = Lazy";
/// If database.sql doesn't exist, then engine is Ordinary
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
}
@ -496,22 +279,20 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const
return ast;
}
void DatabaseOnDisk::drop(const IDatabase & database, const Context & context)
void DatabaseOnDisk::drop(const Context & context)
{
Poco::File(context.getPath() + database.getDataPath()).remove(false);
Poco::File(database.getMetadataPath()).remove(false);
Poco::File(context.getPath() + getDataPath()).remove(false);
Poco::File(getMetadataPath()).remove(false);
}
String DatabaseOnDisk::getObjectMetadataPath(const IDatabase & database, const String & table_name)
String DatabaseOnDisk::getObjectMetadataPath(const String & table_name) const
{
return detail::getObjectMetadataPath(database.getMetadataPath(), table_name);
return getMetadataPath() + escapeForFileName(table_name) + ".sql";
}
time_t DatabaseOnDisk::getObjectMetadataModificationTime(
const IDatabase & database,
const String & table_name)
time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_name) const
{
String table_metadata_path = getObjectMetadataPath(database, table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
Poco::File meta_file(table_metadata_path);
if (meta_file.exists())
@ -520,10 +301,10 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function)
void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
{
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
@ -538,10 +319,10 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(context.getPath() + database.getDataPath() + '/' + object_name).exists())
if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
{
/// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
Poco::File(dir_it->path()).renameTo(database.getMetadataPath() + object_name + ".sql");
Poco::File(dir_it->path()).renameTo(getMetadataPath() + object_name + ".sql");
LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
iterating_function(object_name + ".sql");
}
@ -567,9 +348,64 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
iterating_function(dir_it.name());
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + database.getMetadataPath(),
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
ErrorCodes::INCORRECT_FILE_NAME);
}
}
ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const
{
String query;
try
{
ReadBufferFromFile in(metadata_file_path, METADATA_FILE_BUFFER_SIZE);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (remove_empty && query.empty())
{
LOG_ERROR(log, "File " << metadata_file_path << " is empty. Removing.");
Poco::File(metadata_file_path).remove();
return nullptr;
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
else if (!ast)
return nullptr;
return ast;
}
ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const
{
ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database_name;
}
return ast;
}
}

162
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -11,24 +11,14 @@
namespace DB
{
namespace detail
{
String getObjectMetadataPath(const String & base_path, const String & dictionary_name);
String getDatabaseMetadataPath(const String & base_path);
ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true);
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error);
}
ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger * log);
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path_relative,
const String & table_data_path_relative,
Context & context,
bool has_force_restore_data_flag);
/** Get the row with the table definition based on the CREATE query.
/** Get the string with the table definition based on the CREATE query.
* It is an ATTACH query that you can execute to create a table from the correspondent database.
* See the implementation.
*/
@ -37,147 +27,59 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query);
/* Class to provide basic operations with tables when metadata is stored on disk in .sql files.
*/
class DatabaseOnDisk
class DatabaseOnDisk : public DatabaseWithOwnTablesBase
{
public:
static void createTable(
IDatabase & database,
DatabaseOnDisk(const String & name, const String & metadata_path_, const String & logger)
: DatabaseWithOwnTablesBase(name, logger)
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(database_name) + "/") {}
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query);
const ASTPtr & query) override;
static void createDictionary(
IDatabase & database,
void removeTable(
const Context & context,
const String & dictionary_name,
const ASTPtr & query);
const String & table_name) override;
static void removeTable(
IDatabase & database,
const Context & context,
const String & table_name,
Poco::Logger * log);
static void removeDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
Poco::Logger * log);
template <typename Database>
static void renameTable(
IDatabase & database,
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock);
TableStructureWriteLockHolder & lock) override;
static ASTPtr getCreateTableQuery(
const IDatabase & database,
const Context & context,
const String & table_name);
ASTPtr getCreateDatabaseQuery() const override;
static ASTPtr tryGetCreateTableQuery(
const IDatabase & database,
const Context & context,
const String & table_name);
void drop(const Context & context) override;
static ASTPtr getCreateDictionaryQuery(
const IDatabase & database,
const Context & context,
const String & dictionary_name);
String getObjectMetadataPath(const String & object_name) const override;
static ASTPtr tryGetCreateDictionaryQuery(
const IDatabase & database,
const Context & context,
const String & dictionary_name);
static ASTPtr getCreateDatabaseQuery(
const IDatabase & database,
const Context & context);
static void drop(const IDatabase & database, const Context & context);
static String getObjectMetadataPath(
const IDatabase & database,
const String & object_name);
static time_t getObjectMetadataModificationTime(
const IDatabase & database,
const String & object_name);
time_t getObjectMetadataModificationTime(const String & object_name) const override;
String getDataPath() const override { return data_path; }
String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; }
String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.table); }
String getMetadataPath() const override { return metadata_path; }
protected:
using IteratingFunction = std::function<void(const String &)>;
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function);
void iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const;
private:
static ASTPtr getCreateTableQueryImpl(
const IDatabase & database,
ASTPtr getCreateTableQueryImpl(
const Context & context,
const String & table_name,
bool throw_on_error);
bool throw_on_error) const override;
static ASTPtr getCreateDictionaryQueryImpl(
const IDatabase & database,
const Context & context,
const String & dictionary_name,
bool throw_on_error);
ASTPtr parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const;
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, bool throw_on_error) const;
const String metadata_path;
const String data_path;
};
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int UNKNOWN_TABLE;
extern const int FILE_DOESNT_EXIST;
}
template <typename Database>
void DatabaseOnDisk::renameTable(
IDatabase & database,
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
Database * to_database_concrete = typeid_cast<Database *>(&to_database);
if (!to_database_concrete)
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
StoragePtr table = database.tryGetTable(context, table_name);
if (!table)
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
table->rename("/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/" + escapeForFileName(to_table_name) + '/',
to_database_concrete->getDatabaseName(),
to_table_name, lock);
}
catch (const Exception &)
{
throw;
}
catch (const Poco::Exception & e)
{
/// Better diagnostics.
throw Exception{Exception::CreateFromPoco, e};
}
ASTPtr ast = detail::getQueryFromMetadata(detail::getObjectMetadataPath(database.getMetadataPath(), table_name));
if (!ast)
throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
ast->as<ASTCreateQuery &>().table = to_table_name;
/// NOTE Non-atomic.
to_database_concrete->createTable(context, to_table_name, table, ast);
database.removeTable(context, table_name);
}
}

160
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -1,7 +1,6 @@
#include <iomanip>
#include <Core/Settings.h>
#include <Databases/DatabaseMemory.h>
#include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabaseOrdinary.h>
#include <Databases/DatabasesCommon.h>
@ -15,18 +14,17 @@
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ParserDictionary.h>
#include <Storages/StorageFactory.h>
#include <Dictionaries/DictionaryFactory.h>
#include <Parsers/parseQuery.h>
#include <Parsers/formatAST.h>
#include <Storages/IStorage.h>
#include <Parsers/ASTSetQuery.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Parsers/queryToString.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/Event.h>
#include <Common/Stopwatch.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Common/ThreadPool.h>
#include <Common/escapeForFileName.h>
@ -44,7 +42,6 @@ namespace ErrorCodes
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int CANNOT_PARSE_TEXT;
extern const int EMPTY_LIST_OF_ATTRIBUTES_PASSED;
}
@ -68,7 +65,7 @@ namespace
String table_name;
StoragePtr table;
std::tie(table_name, table)
= createTableFromAST(query, database_name, database.getDataPath(), context, has_force_restore_data_flag);
= createTableFromAST(query, database_name, database.getTableDataPath(query), context, has_force_restore_data_flag);
database.attachTable(table_name, table);
}
catch (const Exception & e)
@ -114,11 +111,8 @@ namespace
}
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context_)
: DatabaseWithOwnTablesBase(std::move(name_))
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(name) + "/")
, log(&Logger::get("DatabaseOrdinary (" + name + ")"))
DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context_)
: DatabaseWithDictionaries(name_, metadata_path_, "DatabaseOrdinary (" + name_ + ")")
{
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -137,12 +131,12 @@ void DatabaseOrdinary::loadStoredObjects(
FileNames file_names;
size_t total_dictionaries = 0;
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [&file_names, &total_dictionaries, this](const String & file_name)
iterateMetadataFiles(context, [&file_names, &total_dictionaries, this](const String & file_name)
{
String full_path = metadata_path + "/" + file_name;
String full_path = getMetadataPath() + file_name;
try
{
auto ast = parseCreateQueryFromMetadataFile(full_path, log);
auto ast = parseQueryFromMetadata(full_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast)
{
auto * create_query = ast->as<ASTCreateQuery>();
@ -237,94 +231,14 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool)
thread_pool.wait();
}
void DatabaseOrdinary::createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query)
{
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
}
void DatabaseOrdinary::createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query)
{
DatabaseOnDisk::createDictionary(*this, context, dictionary_name, query);
}
void DatabaseOrdinary::removeTable(
const Context & context,
const String & table_name)
{
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
void DatabaseOrdinary::removeDictionary(
const Context & context,
const String & table_name)
{
DatabaseOnDisk::removeDictionary(*this, context, table_name, log);
}
void DatabaseOrdinary::renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
DatabaseOnDisk::renameTable<DatabaseOrdinary>(*this, context, table_name, to_database, to_table_name, lock);
}
time_t DatabaseOrdinary::getObjectMetadataModificationTime(
const Context & /* context */,
const String & table_name)
{
return DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name);
}
ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::getCreateDictionaryQuery(const Context & context, const String & dictionary_name) const
{
return DatabaseOnDisk::getCreateDictionaryQuery(*this, context, dictionary_name);
}
ASTPtr DatabaseOrdinary::tryGetCreateDictionaryQuery(const Context & context, const String & dictionary_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, dictionary_name);
}
ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const
{
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
}
void DatabaseOrdinary::alterTable(
const Context & context,
const String & table_name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & storage_modifier)
const StorageInMemoryMetadata & metadata)
{
/// Read the definition of the table and replace the necessary parts with new ones.
String table_name_escaped = escapeForFileName(table_name);
String table_metadata_tmp_path = getMetadataPath() + "/" + table_name_escaped + ".sql.tmp";
String table_metadata_path = getMetadataPath() + "/" + table_name_escaped + ".sql";
String table_metadata_path = getObjectMetadataPath(table_name);
String table_metadata_tmp_path = table_metadata_path + ".tmp";
String statement;
{
@ -338,19 +252,30 @@ void DatabaseOrdinary::alterTable(
const auto & ast_create_query = ast->as<ASTCreateQuery &>();
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns);
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(indices);
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(constraints);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.indices);
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints);
ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
if (storage_modifier)
storage_modifier(*ast_create_query.storage);
ASTStorage & storage_ast = *ast_create_query.storage;
/// ORDER BY may change, but cannot appear, it's required construction
if (metadata.order_by_ast && storage_ast.order_by)
storage_ast.set(storage_ast.order_by, metadata.order_by_ast);
if (metadata.primary_key_ast)
storage_ast.set(storage_ast.primary_key, metadata.primary_key_ast);
if (metadata.ttl_for_table_ast)
storage_ast.set(storage_ast.ttl_table, metadata.ttl_for_table_ast);
if (metadata.settings_ast)
storage_ast.set(storage_ast.settings, metadata.settings_ast);
statement = getObjectDefinitionFromCreateQuery(ast);
{
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
@ -372,31 +297,4 @@ void DatabaseOrdinary::alterTable(
}
}
void DatabaseOrdinary::drop(const Context & context)
{
DatabaseOnDisk::drop(*this, context);
}
String DatabaseOrdinary::getDataPath() const
{
return data_path;
}
String DatabaseOrdinary::getMetadataPath() const
{
return metadata_path;
}
String DatabaseOrdinary::getDatabaseName() const
{
return name;
}
String DatabaseOrdinary::getObjectMetadataPath(const String & table_name) const
{
return DatabaseOnDisk::getObjectMetadataPath(*this, table_name);
}
}

69
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Databases/DatabaseWithDictionaries.h>
#include <Common/ThreadPool.h>
@ -11,10 +11,10 @@ namespace DB
* It stores tables list in filesystem using list of .sql files,
* that contain declaration of table represented by SQL ATTACH TABLE query.
*/
class DatabaseOrdinary : public DatabaseWithOwnTablesBase
class DatabaseOrdinary : public DatabaseWithDictionaries //DatabaseWithOwnTablesBase
{
public:
DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context);
DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context);
String getEngineName() const override { return "Ordinary"; }
@ -22,73 +22,12 @@ public:
Context & context,
bool has_force_restore_data_flag) override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder &) override;
void alterTable(
const Context & context,
const String & name,
const ColumnsDescription & columns,
const IndicesDescription & indices,
const ConstraintsDescription & constraints,
const ASTModifier & engine_modifier) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateDictionaryQuery(
const Context & context,
const String & name) const override;
ASTPtr getCreateDictionaryQuery(
const Context & context,
const String & name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
String getDataPath() const override;
String getDatabaseName() const override;
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop(const Context & context) override;
const StorageInMemoryMetadata & metadata) override;
private:
const String metadata_path;
const String data_path;
Poco::Logger * log;
void startupTables(ThreadPool & thread_pool);
};

256
dbms/src/Databases/DatabaseWithDictionaries.cpp Normal file
View File

@ -0,0 +1,256 @@
#include <Databases/DatabaseWithDictionaries.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Storages/StorageDictionary.h>
#include <IO/WriteBufferFromFile.h>
#include <Poco/File.h>
#include <ext/scope_guard.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int TABLE_ALREADY_EXISTS;
extern const int UNKNOWN_TABLE;
extern const int LOGICAL_ERROR;
extern const int DICTIONARY_ALREADY_EXISTS;
}
void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithDictionaries::createDictionary(const Context & context, const String & dictionary_name, const ASTPtr & query)
{
const auto & settings = context.getSettingsRef();
/** The code is based on the assumption that all threads share the same order of operations:
* - create the .sql.tmp file;
* - add the dictionary to ExternalDictionariesLoader;
* - load the dictionary in case dictionaries_lazy_load == false;
* - attach the dictionary;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a dictionary with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (isDictionaryExist(context, dictionary_name))
throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (isTableExist(context, dictionary_name))
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp";
String statement = getObjectDefinitionFromCreateQuery(query);
{
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
bool succeeded = false;
SCOPE_EXIT({
if (!succeeded)
Poco::File(dictionary_metadata_tmp_path).remove();
});
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
{
/// load() is called here to force loading the dictionary, wait until the loading is finished,
/// and throw an exception if the loading is failed.
external_loader.load(full_name);
}
attachDictionary(dictionary_name, context);
SCOPE_EXIT({
if (!succeeded)
detachDictionary(dictionary_name, context);
});
/// If it was ATTACH query and file with dictionary metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path);
/// ExternalDictionariesLoader doesn't know we renamed the metadata path.
/// So we have to manually call reloadConfig() here.
external_loader.reloadConfig(getDatabaseName(), full_name);
/// Everything's ok.
succeeded = true;
}
void DatabaseWithDictionaries::removeDictionary(const Context & context, const String & dictionary_name)
{
detachDictionary(dictionary_name, context);
String dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
try
{
Poco::File(dictionary_metadata_path).remove();
}
catch (...)
{
/// If remove was not possible for some reason
attachDictionary(dictionary_name, context);
throw;
}
}
StoragePtr DatabaseWithDictionaries::tryGetTable(const Context & context, const String & table_name) const
{
if (auto table_ptr = DatabaseWithOwnTablesBase::tryGetTable(context, table_name))
return table_ptr;
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name);
return {};
}
DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
/// NOTE: it's not atomic
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name);
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithDictionaries::isDictionaryExist(const Context & /*context*/, const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & context, const String & table_name) const
{
auto dict_name = database_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(database_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl(
const Context & context,
const String & dictionary_name,
bool throw_on_error) const
{
ASTPtr ast;
auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
ast = getCreateQueryFromMetadata(dictionary_metadata_path, throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_dictionary = isDictionaryExist(context, dictionary_name);
auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
return ast;
}
}

39
dbms/src/Databases/DatabaseWithDictionaries.h Normal file
View File

@ -0,0 +1,39 @@
#include <Databases/DatabaseOnDisk.h>
namespace DB
{
class DatabaseWithDictionaries : public DatabaseOnDisk
{
public:
void attachDictionary(const String & name, const Context & context) override;
void detachDictionary(const String & name, const Context & context) override;
void createDictionary(const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeDictionary(const Context & context, const String & dictionary_name) override;
StoragePtr tryGetTable(const Context & context, const String & table_name) const override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
protected:
DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger)
: DatabaseOnDisk(name, metadata_path_, logger) {}
StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const;
ASTPtr getCreateDictionaryQueryImpl(const Context & context,
const String & dictionary_name,
bool throw_on_error) const override;
};
}

125
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,6 +1,4 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -22,23 +20,9 @@ namespace ErrorCodes
extern const int DICTIONARY_ALREADY_EXISTS;
}
namespace
DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const String & logger)
: IDatabase(name_), log(&Logger::get(logger))
{
StoragePtr getDictionaryStorage(const Context & context, const String & table_name, const String & db_name)
{
auto dict_name = db_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(db_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
}
bool DatabaseWithOwnTablesBase::isTableExist(
@ -49,57 +33,17 @@ bool DatabaseWithOwnTablesBase::isTableExist(
return tables.find(table_name) != tables.end() || dictionaries.find(table_name) != dictionaries.end();
}
bool DatabaseWithOwnTablesBase::isDictionaryExist(
const Context & /*context*/,
const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithOwnTablesBase::tryGetTable(
const Context & context,
const Context & /*context*/,
const String & table_name) const
{
{
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
}
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name, getDatabaseName());
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
return {};
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name, getDatabaseName());
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(mutex);
@ -114,20 +58,6 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Con
return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(filtered_tables));
}
DatabaseDictionariesIteratorPtr DatabaseWithOwnTablesBase::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const
{
std::lock_guard lock(mutex);
@ -140,11 +70,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
{
std::lock_guard lock(mutex);
if (dictionaries.count(table_name))
throw Exception("Cannot detach dictionary " + name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE);
throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE);
auto it = tables.find(table_name);
if (it == tables.end())
throw Exception("Table " + backQuote(name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
res = it->second;
tables.erase(it);
}
@ -152,44 +82,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
return res;
}
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table)
{
std::lock_guard lock(mutex);
if (!tables.emplace(table_name, table).second)
throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
}
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
}
void DatabaseWithOwnTablesBase::shutdown()
@ -217,7 +114,7 @@ DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
{
try
{
shutdown();
DatabaseWithOwnTablesBase::shutdown();
}
catch (...)
{

15
dbms/src/Databases/DatabasesCommon.h
View File

@ -23,8 +23,6 @@ public:
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
@ -33,30 +31,21 @@ public:
void attachTable(const String & table_name, const StoragePtr & table) override;
void attachDictionary(const String & name, const Context & context) override;
StoragePtr detachTable(const String & table_name) override;
void detachDictionary(const String & name, const Context & context) override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void shutdown() override;
virtual ~DatabaseWithOwnTablesBase() override;
protected:
String name;
mutable std::mutex mutex;
Tables tables;
Dictionaries dictionaries;
Poco::Logger * log;
DatabaseWithOwnTablesBase(String name_) : name(std::move(name_)) { }
DatabaseWithOwnTablesBase(const String & name_, const String & logger);
};
}

139
dbms/src/Databases/IDatabase.h
View File

@ -3,6 +3,7 @@
#include <Core/Types.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/IStorage_fwd.h>
#include <Storages/StorageInMemoryMetadata.h>
#include <Dictionaries/IDictionary.h>
#include <Common/Exception.h>
@ -20,11 +21,15 @@ struct ConstraintsDescription;
class ColumnsDescription;
struct IndicesDescription;
struct TableStructureWriteLockHolder;
class ASTCreateQuery;
using Dictionaries = std::set<String>;
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
}
class IDatabaseTablesIterator
@ -96,14 +101,15 @@ using DatabaseDictionariesIteratorPtr = std::unique_ptr<DatabaseDictionariesSnap
class IDatabase : public std::enable_shared_from_this<IDatabase>
{
public:
IDatabase() = delete;
IDatabase(String database_name_) : database_name(std::move(database_name_)) {}
/// Get name of database engine.
virtual String getEngineName() const = 0;
/// Load a set of existing tables.
/// You can call only once, right after the object is created.
virtual void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) = 0;
virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/) {}
/// Check the existence of the table.
virtual bool isTableExist(
@ -112,8 +118,11 @@ public:
/// Check the existence of the dictionary
virtual bool isDictionaryExist(
const Context & context,
const String & name) const = 0;
const Context & /*context*/,
const String & /*name*/) const
{
return false;
}
/// Get the table for work. Return nullptr if there is no table.
virtual StoragePtr tryGetTable(
@ -127,7 +136,10 @@ public:
virtual DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) = 0;
/// Get an iterator to pass through all the dictionaries.
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) = 0;
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & /*context*/, [[maybe_unused]] const FilterByNameFunction & filter_by_dictionary_name = {})
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
/// Get an iterator to pass through all the tables and dictionary tables.
virtual DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name = {})
@ -140,39 +152,63 @@ public:
/// Add the table to the database. Record its presence in the metadata.
virtual void createTable(
const Context & context,
const String & name,
const StoragePtr & table,
const ASTPtr & query) = 0;
const Context & /*context*/,
const String & /*name*/,
const StoragePtr & /*table*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add the dictionary to the database. Record its presence in the metadata.
virtual void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) = 0;
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Delete the table from the database. Delete the metadata.
virtual void removeTable(
const Context & context,
const String & name) = 0;
const Context & /*context*/,
const String & /*name*/)
{
throw Exception("There is no DROP TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Delete the dictionary from the database. Delete the metadata.
virtual void removeDictionary(
const Context & context,
const String & dictionary_name) = 0;
const Context & /*context*/,
const String & /*dictionary_name*/)
{
throw Exception("There is no DROP DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add a table to the database, but do not add it to the metadata. The database may not support this method.
virtual void attachTable(const String & name, const StoragePtr & table) = 0;
virtual void attachTable(const String & /*name*/, const StoragePtr & /*table*/)
{
throw Exception("There is no ATTACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add dictionary to the database, but do not add it to the metadata. The database may not support this method.
/// If dictionaries_lazy_load is false it also starts loading the dictionary asynchronously.
virtual void attachDictionary(const String & name, const Context & context) = 0;
virtual void attachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no ATTACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Forget about the table without deleting it, and return it. The database may not support this method.
virtual StoragePtr detachTable(const String & name) = 0;
virtual StoragePtr detachTable(const String & /*name*/)
{
throw Exception("There is no DETACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Forget about the dictionary without deleting it. The database may not support this method.
virtual void detachDictionary(const String & name, const Context & context) = 0;
virtual void detachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no DETACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Rename the table and possibly move the table to another database.
virtual void renameTable(
@ -192,42 +228,50 @@ public:
virtual void alterTable(
const Context & /*context*/,
const String & /*name*/,
const ColumnsDescription & /*columns*/,
const IndicesDescription & /*indices*/,
const ConstraintsDescription & /*constraints*/,
const ASTModifier & /*engine_modifier*/)
const StorageInMemoryMetadata & /*metadata*/)
{
throw Exception(getEngineName() + ": renameTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
throw Exception(getEngineName() + ": alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
/// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
virtual time_t getObjectMetadataModificationTime(
const Context & context,
const String & name) = 0;
virtual time_t getObjectMetadataModificationTime(const String & /*name*/) const
{
return static_cast<time_t>(0);
}
/// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
virtual ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const = 0;
virtual ASTPtr getCreateTableQuery(const Context & context, const String & name) const
ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const noexcept
{
return tryGetCreateTableQuery(context, name);
return getCreateTableQueryImpl(context, name, false);
}
ASTPtr getCreateTableQuery(const Context & context, const String & name) const
{
return getCreateTableQueryImpl(context, name, true);
}
/// Get the CREATE DICTIONARY query for the dictionary. Returns nullptr if dictionary doesn't exists.
virtual ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & name) const = 0;
virtual ASTPtr getCreateDictionaryQuery(const Context & context, const String & name) const
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & name) const noexcept
{
return tryGetCreateDictionaryQuery(context, name);
return getCreateDictionaryQueryImpl(context, name, false);
}
ASTPtr getCreateDictionaryQuery(const Context & context, const String & name) const
{
return getCreateDictionaryQueryImpl(context, name, true);
}
/// Get the CREATE DATABASE query for current database.
virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
virtual ASTPtr getCreateDatabaseQuery() const = 0;
/// Get name of database.
virtual String getDatabaseName() const = 0;
String getDatabaseName() const { return database_name; }
/// Returns path for persistent data storage if the database supports it, empty string otherwise
virtual String getDataPath() const { return {}; }
/// Returns path for persistent data storage for table if the database supports it, empty string otherwise. Table must exist
virtual String getTableDataPath(const String & /*table_name*/) const { return {}; }
/// Returns path for persistent data storage for CREATE/ATTACH query if the database supports it, empty string otherwise
virtual String getTableDataPath(const ASTCreateQuery & /*query*/) const { return {}; }
/// Returns metadata path if the database supports it, empty string otherwise
virtual String getMetadataPath() const { return {}; }
/// Returns metadata path of a concrete table if the database supports it, empty string otherwise
@ -240,6 +284,23 @@ public:
virtual void drop(const Context & /*context*/) {}
virtual ~IDatabase() {}
protected:
virtual ASTPtr getCreateTableQueryImpl(const Context & /*context*/, const String & /*name*/, bool throw_on_error) const
{
if (throw_on_error)
throw Exception("There is no SHOW CREATE TABLE query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
return nullptr;
}
virtual ASTPtr getCreateDictionaryQueryImpl(const Context & /*context*/, const String & /*name*/, bool throw_on_error) const
{
if (throw_on_error)
throw Exception("There is no SHOW CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
return nullptr;
}
String database_name;
};
using DatabasePtr = std::shared_ptr<IDatabase>;

24
dbms/src/Disks/DiskSpaceMonitor.cpp
View File

@ -208,6 +208,30 @@ StoragePolicy::StoragePolicy(String name_, Volumes volumes_, double move_factor_
}
bool StoragePolicy::isDefaultPolicy() const
{
/// Guessing if this policy is default, not 100% correct though.
if (getName() != "default")
return false;
if (volumes.size() != 1)
return false;
if (volumes[0]->getName() != "default")
return false;
const auto & disks = volumes[0]->disks;
if (disks.size() != 1)
return false;
if (disks[0]->getName() != "default")
return false;
return true;
}
Disks StoragePolicy::getDisks() const
{
Disks res;

2
dbms/src/Disks/DiskSpaceMonitor.h
View File

@ -103,6 +103,8 @@ public:
StoragePolicy(String name_, Volumes volumes_, double move_factor_);
bool isDefaultPolicy() const;
/// Returns disks ordered by volumes priority
Disks getDisks() const;

2
dbms/src/Functions/CRC.cpp
View File

@ -2,7 +2,7 @@
#include <DataTypes/DataTypeString.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionStringOrArrayToT.h>
#include "registerFunctions.h"
namespace
{

2
dbms/src/Functions/FunctionFQDN.cpp
View File

@ -3,7 +3,7 @@
#include <DataTypes/DataTypeString.h>
#include <Common/getFQDNOrHostName.h>
#include <Core/Field.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionFactory.h
View File

@ -2,8 +2,6 @@
#include <Functions/IFunctionAdaptors.h>
#include <Common/IFactoryWithAliases.h>
#include "URL/registerFunctionsURL.h"
#include "registerFunctions.h"
#include <functional>
#include <memory>

1
dbms/src/Functions/FunctionJoinGet.cpp
View File

@ -5,7 +5,6 @@
#include <Interpreters/Context.h>
#include <Interpreters/Join.h>
#include <Storages/StorageJoin.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/FunctionsBitmap.cpp
View File

@ -1,5 +1,4 @@
#include <Functions/FunctionFactory.h>
#include "registerFunctions.h"
// TODO include this last because of a broken roaring header. See the comment
// inside.

2
dbms/src/Functions/FunctionsCoding.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsCoding.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsConversion.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsConversion.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsEmbeddedDictionaries.cpp
View File

@ -1,6 +1,6 @@
#include "FunctionFactory.h"
#include "FunctionsEmbeddedDictionaries.h"
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsExternalDictionaries.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsExternalDictionaries.h>
#include "registerFunctions.h"
namespace DB
{

1
dbms/src/Functions/FunctionsExternalModels.cpp
View File

@ -15,7 +15,6 @@
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypeTuple.h>
#include <Common/assert_cast.h>
#include "registerFunctions.h"
namespace DB

2
dbms/src/Functions/FunctionsFindCluster.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsFindCluster.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsFormatting.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsFormatting.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsHashing.cpp
View File

@ -1,7 +1,7 @@
#include "FunctionsHashing.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctions.h"
namespace DB
{

1
dbms/src/Functions/FunctionsJSON.cpp
View File

@ -1,6 +1,5 @@
#include <Functions/FunctionsJSON.h>
#include <Functions/FunctionFactory.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/FunctionsRandom.cpp
View File

@ -3,7 +3,6 @@
#include <Common/HashTable/Hash.h>
#include <Common/randomSeed.h>
#include <common/unaligned.h>
#include "registerFunctions.h"
namespace DB

2
dbms/src/Functions/FunctionsReinterpret.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsReinterpret.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsRound.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsRound.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsStringArray.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsStringArray.h>
#include "registerFunctions.h"
namespace DB
{

2
dbms/src/Functions/FunctionsStringRegex.cpp
View File

@ -1,6 +1,4 @@
#include "FunctionsStringRegex.h"
#include "registerFunctions.h"
#include "FunctionsStringSearch.h"
#include <Columns/ColumnFixedString.h>
#include <DataTypes/DataTypeFixedString.h>

1
dbms/src/Functions/FunctionsStringSearch.cpp
View File

@ -1,5 +1,4 @@
#include "FunctionsStringSearch.h"
#include "registerFunctions.h"
#include <algorithm>
#include <memory>

2
dbms/src/Functions/FunctionsStringSimilarity.cpp
View File

@ -1,6 +1,4 @@
#include <Functions/FunctionsStringSimilarity.h>
#include "registerFunctions.h"
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsHashing.h>
#include <Common/HashTable/ClearableHashMap.h>

5
dbms/src/Functions/GeoUtils.h
View File

@ -261,6 +261,10 @@ void PointInPolygonWithGrid<CoordinateType>::buildGrid()
for (size_t row = 0; row < grid_size; ++row)
{
#pragma GCC diagnostic push
#if !__clang__
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
CoordinateType y_min = min_corner.y() + row * cell_height;
CoordinateType y_max = min_corner.y() + (row + 1) * cell_height;
@ -268,6 +272,7 @@ void PointInPolygonWithGrid<CoordinateType>::buildGrid()
{
CoordinateType x_min = min_corner.x() + col * cell_width;
CoordinateType x_max = min_corner.x() + (col + 1) * cell_width;
#pragma GCC diagnostic pop
Box cell_box(Point(x_min, y_min), Point(x_max, y_max));
Polygon cell_bound;

1
dbms/src/Functions/URL/basename.cpp
View File

@ -1,6 +1,5 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionStringToString.h>
#include <Functions/registerFunctions.h>
#include <common/find_symbols.h>
#include "FunctionsURL.h"

27
dbms/src/Functions/URL/registerFunctionsURL.cpp
View File

@ -1,8 +1,31 @@
#include "registerFunctionsURL.h"
namespace DB
{
class FunctionFactory;
void registerFunctionProtocol(FunctionFactory & factory);
void registerFunctionDomain(FunctionFactory & factory);
void registerFunctionDomainWithoutWWW(FunctionFactory & factory);
void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory);
void registerFunctionTopLevelDomain(FunctionFactory & factory);
void registerFunctionPath(FunctionFactory & factory);
void registerFunctionPathFull(FunctionFactory & factory);
void registerFunctionQueryString(FunctionFactory & factory);
void registerFunctionFragment(FunctionFactory & factory);
void registerFunctionQueryStringAndFragment(FunctionFactory & factory);
void registerFunctionExtractURLParameter(FunctionFactory & factory);
void registerFunctionExtractURLParameters(FunctionFactory & factory);
void registerFunctionExtractURLParameterNames(FunctionFactory & factory);
void registerFunctionURLHierarchy(FunctionFactory & factory);
void registerFunctionURLPathHierarchy(FunctionFactory & factory);
void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory & factory);
void registerFunctionCutWWW(FunctionFactory & factory);
void registerFunctionCutQueryString(FunctionFactory & factory);
void registerFunctionCutFragment(FunctionFactory & factory);
void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory);
void registerFunctionCutURLParameter(FunctionFactory & factory);
void registerFunctionDecodeURLComponent(FunctionFactory & factory);
void registerFunctionsURL(FunctionFactory & factory)
{
registerFunctionProtocol(factory);

32
dbms/src/Functions/URL/registerFunctionsURL.h
View File

@ -1,32 +0,0 @@
#pragma once
namespace DB
{
class FunctionFactory;
void registerFunctionProtocol(FunctionFactory &);
void registerFunctionDomain(FunctionFactory &);
void registerFunctionDomainWithoutWWW(FunctionFactory &);
void registerFunctionFirstSignificantSubdomain(FunctionFactory &);
void registerFunctionTopLevelDomain(FunctionFactory &);
void registerFunctionPath(FunctionFactory &);
void registerFunctionPathFull(FunctionFactory &);
void registerFunctionQueryString(FunctionFactory &);
void registerFunctionFragment(FunctionFactory &);
void registerFunctionQueryStringAndFragment(FunctionFactory &);
void registerFunctionExtractURLParameter(FunctionFactory &);
void registerFunctionExtractURLParameters(FunctionFactory &);
void registerFunctionExtractURLParameterNames(FunctionFactory &);
void registerFunctionURLHierarchy(FunctionFactory &);
void registerFunctionURLPathHierarchy(FunctionFactory &);
void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory &);
void registerFunctionCutWWW(FunctionFactory &);
void registerFunctionCutQueryString(FunctionFactory &);
void registerFunctionCutFragment(FunctionFactory &);
void registerFunctionCutQueryStringAndFragment(FunctionFactory &);
void registerFunctionCutURLParameter(FunctionFactory &);
void registerFunctionDecodeURLComponent(FunctionFactory &);
void registerFunctionsURL(FunctionFactory &);
}

2
dbms/src/Functions/acos.cpp
View File

@ -1,6 +1,6 @@
#include <Functions/FunctionMathUnary.h>
#include <Functions/FunctionFactory.h>
#include "registerFunctions.h"
namespace DB
{

1
dbms/src/Functions/addDays.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addHours.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addMinutes.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addMonths.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addQuarters.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addSeconds.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addWeeks.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addYears.cpp
View File

@ -1,7 +1,6 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionDateOrDateTimeAddInterval.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addressToLine.cpp
View File

@ -18,7 +18,6 @@
#include <mutex>
#include <filesystem>
#include <unordered_map>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/addressToSymbol.cpp
View File

@ -9,7 +9,6 @@
#include <Functions/FunctionFactory.h>
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/appendTrailingCharIfAbsent.cpp
View File

@ -5,7 +5,6 @@
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunctionImpl.h>
#include <ext/range.h>
#include "registerFunctions.h"
namespace DB

1
dbms/src/Functions/array/array.cpp
View File

@ -4,7 +4,6 @@
#include <DataTypes/getLeastSupertype.h>
#include <Columns/ColumnArray.h>
#include <Interpreters/castColumn.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayAll.cpp
View File

@ -2,7 +2,6 @@
#include <Columns/ColumnsNumber.h>
#include "FunctionArrayMapped.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayCompact.cpp
View File

@ -4,7 +4,6 @@
#include <Columns/ColumnDecimal.h>
#include <Functions/array/FunctionArrayMapped.h>
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayConcat.cpp
View File

@ -8,7 +8,6 @@
#include <Columns/ColumnConst.h>
#include <Common/typeid_cast.h>
#include <ext/range.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayCount.cpp
View File

@ -2,7 +2,6 @@
#include <Columns/ColumnsNumber.h>
#include "FunctionArrayMapped.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayCumSum.cpp
View File

@ -4,7 +4,6 @@
#include <Columns/ColumnDecimal.h>
#include "FunctionArrayMapped.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayCumSumNonNegative.cpp
View File

@ -4,7 +4,6 @@
#include <Columns/ColumnDecimal.h>
#include "FunctionArrayMapped.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayDifference.cpp
View File

@ -4,7 +4,6 @@
#include <Columns/ColumnDecimal.h>
#include "FunctionArrayMapped.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayDistinct.cpp
View File

@ -9,7 +9,6 @@
#include <Common/HashTable/ClearableHashSet.h>
#include <Common/SipHash.h>
#include <Common/assert_cast.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayElement.cpp
View File

@ -12,7 +12,6 @@
#include <Columns/ColumnTuple.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayEnumerate.cpp
View File

@ -5,7 +5,6 @@
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include "registerFunctionsArray.h"
namespace DB

1
dbms/src/Functions/array/arrayEnumerateDense.cpp
View File

@ -1,6 +1,5 @@
#include "arrayEnumerateExtended.h"
#include <Functions/FunctionFactory.h>
#include "registerFunctionsArray.h"
namespace DB

Some files were not shown because too many files have changed in this diff Show More