Big integers and UUID in dictionaries

This commit is contained in:
Alexey Milovidov 2021-05-08 22:01:59 +03:00
parent 49c0e18e45
commit 49160ae1ba
8 changed files with 73 additions and 9 deletions

View File

@ -594,16 +594,20 @@ private:
PaddedPODArray<UInt32>, PaddedPODArray<UInt32>,
PaddedPODArray<UInt64>, PaddedPODArray<UInt64>,
PaddedPODArray<UInt128>, PaddedPODArray<UInt128>,
PaddedPODArray<UInt256>,
PaddedPODArray<Int8>, PaddedPODArray<Int8>,
PaddedPODArray<Int16>, PaddedPODArray<Int16>,
PaddedPODArray<Int32>, PaddedPODArray<Int32>,
PaddedPODArray<Int64>, PaddedPODArray<Int64>,
PaddedPODArray<Int128>,
PaddedPODArray<Int256>,
PaddedPODArray<Decimal32>, PaddedPODArray<Decimal32>,
PaddedPODArray<Decimal64>, PaddedPODArray<Decimal64>,
PaddedPODArray<Decimal128>, PaddedPODArray<Decimal128>,
PaddedPODArray<Decimal256>, PaddedPODArray<Decimal256>,
PaddedPODArray<Float32>, PaddedPODArray<Float32>,
PaddedPODArray<Float64>, PaddedPODArray<Float64>,
PaddedPODArray<UUID>,
PaddedPODArray<StringRef>, PaddedPODArray<StringRef>,
std::vector<Field>> attribute_container; std::vector<Field>> attribute_container;
}; };

View File

@ -53,11 +53,14 @@ AttributeUnderlyingType getAttributeUnderlyingType(const DataTypePtr & type)
case TypeIndex::UInt32: return AttributeUnderlyingType::utUInt32; case TypeIndex::UInt32: return AttributeUnderlyingType::utUInt32;
case TypeIndex::UInt64: return AttributeUnderlyingType::utUInt64; case TypeIndex::UInt64: return AttributeUnderlyingType::utUInt64;
case TypeIndex::UInt128: return AttributeUnderlyingType::utUInt128; case TypeIndex::UInt128: return AttributeUnderlyingType::utUInt128;
case TypeIndex::UInt256: return AttributeUnderlyingType::utUInt256;
case TypeIndex::Int8: return AttributeUnderlyingType::utInt8; case TypeIndex::Int8: return AttributeUnderlyingType::utInt8;
case TypeIndex::Int16: return AttributeUnderlyingType::utInt16; case TypeIndex::Int16: return AttributeUnderlyingType::utInt16;
case TypeIndex::Int32: return AttributeUnderlyingType::utInt32; case TypeIndex::Int32: return AttributeUnderlyingType::utInt32;
case TypeIndex::Int64: return AttributeUnderlyingType::utInt64; case TypeIndex::Int64: return AttributeUnderlyingType::utInt64;
case TypeIndex::Int128: return AttributeUnderlyingType::utInt128;
case TypeIndex::Int256: return AttributeUnderlyingType::utInt256;
case TypeIndex::Float32: return AttributeUnderlyingType::utFloat32; case TypeIndex::Float32: return AttributeUnderlyingType::utFloat32;
case TypeIndex::Float64: return AttributeUnderlyingType::utFloat64; case TypeIndex::Float64: return AttributeUnderlyingType::utFloat64;
@ -71,7 +74,7 @@ AttributeUnderlyingType getAttributeUnderlyingType(const DataTypePtr & type)
case TypeIndex::DateTime: return AttributeUnderlyingType::utUInt32; case TypeIndex::DateTime: return AttributeUnderlyingType::utUInt32;
case TypeIndex::DateTime64: return AttributeUnderlyingType::utUInt64; case TypeIndex::DateTime64: return AttributeUnderlyingType::utUInt64;
case TypeIndex::UUID: return AttributeUnderlyingType::utUInt128; case TypeIndex::UUID: return AttributeUnderlyingType::utUUID;
case TypeIndex::String: return AttributeUnderlyingType::utString; case TypeIndex::String: return AttributeUnderlyingType::utString;
@ -99,7 +102,9 @@ std::string toString(AttributeUnderlyingType type)
case AttributeUnderlyingType::utUInt64: case AttributeUnderlyingType::utUInt64:
return "UInt64"; return "UInt64";
case AttributeUnderlyingType::utUInt128: case AttributeUnderlyingType::utUInt128:
return "UUID"; return "UInt128";
case AttributeUnderlyingType::utUInt256:
return "UInt256";
case AttributeUnderlyingType::utInt8: case AttributeUnderlyingType::utInt8:
return "Int8"; return "Int8";
case AttributeUnderlyingType::utInt16: case AttributeUnderlyingType::utInt16:
@ -108,6 +113,10 @@ std::string toString(AttributeUnderlyingType type)
return "Int32"; return "Int32";
case AttributeUnderlyingType::utInt64: case AttributeUnderlyingType::utInt64:
return "Int64"; return "Int64";
case AttributeUnderlyingType::utInt128:
return "Int128";
case AttributeUnderlyingType::utInt256:
return "Int256";
case AttributeUnderlyingType::utFloat32: case AttributeUnderlyingType::utFloat32:
return "Float32"; return "Float32";
case AttributeUnderlyingType::utFloat64: case AttributeUnderlyingType::utFloat64:
@ -120,6 +129,8 @@ std::string toString(AttributeUnderlyingType type)
return "Decimal128"; return "Decimal128";
case AttributeUnderlyingType::utDecimal256: case AttributeUnderlyingType::utDecimal256:
return "Decimal256"; return "Decimal256";
case AttributeUnderlyingType::utUUID:
return "UUID";
case AttributeUnderlyingType::utString: case AttributeUnderlyingType::utString:
return "String"; return "String";
} }

View File

@ -23,16 +23,20 @@ enum class AttributeUnderlyingType
utUInt32, utUInt32,
utUInt64, utUInt64,
utUInt128, utUInt128,
utUInt256,
utInt8, utInt8,
utInt16, utInt16,
utInt32, utInt32,
utInt64, utInt64,
utInt128,
utInt256,
utFloat32, utFloat32,
utFloat64, utFloat64,
utDecimal32, utDecimal32,
utDecimal64, utDecimal64,
utDecimal128, utDecimal128,
utDecimal256, utDecimal256,
utUUID,
utString utString
}; };
@ -96,6 +100,9 @@ void callOnDictionaryAttributeType(AttributeUnderlyingType type, F&& func)
case AttributeUnderlyingType::utUInt128: case AttributeUnderlyingType::utUInt128:
func(DictionaryAttributeType<UInt128>()); func(DictionaryAttributeType<UInt128>());
break; break;
case AttributeUnderlyingType::utUInt256:
func(DictionaryAttributeType<UInt128>());
break;
case AttributeUnderlyingType::utInt8: case AttributeUnderlyingType::utInt8:
func(DictionaryAttributeType<Int8>()); func(DictionaryAttributeType<Int8>());
break; break;
@ -108,6 +115,12 @@ void callOnDictionaryAttributeType(AttributeUnderlyingType type, F&& func)
case AttributeUnderlyingType::utInt64: case AttributeUnderlyingType::utInt64:
func(DictionaryAttributeType<Int64>()); func(DictionaryAttributeType<Int64>());
break; break;
case AttributeUnderlyingType::utInt128:
func(DictionaryAttributeType<Int64>());
break;
case AttributeUnderlyingType::utInt256:
func(DictionaryAttributeType<Int64>());
break;
case AttributeUnderlyingType::utFloat32: case AttributeUnderlyingType::utFloat32:
func(DictionaryAttributeType<Float32>()); func(DictionaryAttributeType<Float32>());
break; break;
@ -129,6 +142,9 @@ void callOnDictionaryAttributeType(AttributeUnderlyingType type, F&& func)
case AttributeUnderlyingType::utDecimal256: case AttributeUnderlyingType::utDecimal256:
func(DictionaryAttributeType<Decimal256>()); func(DictionaryAttributeType<Decimal256>());
break; break;
case AttributeUnderlyingType::utUUID:
func(DictionaryAttributeType<UUID>());
break;
} }
}; };

View File

@ -113,16 +113,20 @@ private:
UInt32, UInt32,
UInt64, UInt64,
UInt128, UInt128,
UInt256,
Int8, Int8,
Int16, Int16,
Int32, Int32,
Int64, Int64,
Int128,
Int256,
Decimal32, Decimal32,
Decimal64, Decimal64,
Decimal128, Decimal128,
Decimal256, Decimal256,
Float32, Float32,
Float64, Float64,
UUID,
StringRef> StringRef>
null_values; null_values;
std::variant< std::variant<
@ -131,16 +135,20 @@ private:
ContainerType<UInt32>, ContainerType<UInt32>,
ContainerType<UInt64>, ContainerType<UInt64>,
ContainerType<UInt128>, ContainerType<UInt128>,
ContainerType<UInt256>,
ContainerType<Int8>, ContainerType<Int8>,
ContainerType<Int16>, ContainerType<Int16>,
ContainerType<Int32>, ContainerType<Int32>,
ContainerType<Int64>, ContainerType<Int64>,
ContainerType<Int128>,
ContainerType<Int256>,
ContainerType<Decimal32>, ContainerType<Decimal32>,
ContainerType<Decimal64>, ContainerType<Decimal64>,
ContainerType<Decimal128>, ContainerType<Decimal128>,
ContainerType<Decimal256>, ContainerType<Decimal256>,
ContainerType<Float32>, ContainerType<Float32>,
ContainerType<Float64>, ContainerType<Float64>,
ContainerType<UUID>,
ContainerType<StringRef>> ContainerType<StringRef>>
container; container;

View File

@ -145,16 +145,20 @@ private:
UInt32, UInt32,
UInt64, UInt64,
UInt128, UInt128,
UInt256,
Int8, Int8,
Int16, Int16,
Int32, Int32,
Int64, Int64,
Int128,
Int256,
Decimal32, Decimal32,
Decimal64, Decimal64,
Decimal128, Decimal128,
Decimal256, Decimal256,
Float32, Float32,
Float64, Float64,
UUID,
StringRef> StringRef>
null_values; null_values;
@ -164,16 +168,20 @@ private:
CollectionType<UInt32>, CollectionType<UInt32>,
CollectionType<UInt64>, CollectionType<UInt64>,
CollectionType<UInt128>, CollectionType<UInt128>,
CollectionType<UInt256>,
CollectionType<Int8>, CollectionType<Int8>,
CollectionType<Int16>, CollectionType<Int16>,
CollectionType<Int32>, CollectionType<Int32>,
CollectionType<Int64>, CollectionType<Int64>,
CollectionType<Int128>,
CollectionType<Int256>,
CollectionType<Decimal32>, CollectionType<Decimal32>,
CollectionType<Decimal64>, CollectionType<Decimal64>,
CollectionType<Decimal128>, CollectionType<Decimal128>,
CollectionType<Decimal256>, CollectionType<Decimal256>,
CollectionType<Float32>, CollectionType<Float32>,
CollectionType<Float64>, CollectionType<Float64>,
CollectionType<UUID>,
CollectionType<StringRef>> CollectionType<StringRef>>
container; container;

View File

@ -94,16 +94,20 @@ private:
UInt32, UInt32,
UInt64, UInt64,
UInt128, UInt128,
UInt256,
Int8, Int8,
Int16, Int16,
Int32, Int32,
Int64, Int64,
Int128,
Int256,
Decimal32, Decimal32,
Decimal64, Decimal64,
Decimal128, Decimal128,
Decimal256, Decimal256,
Float32, Float32,
Float64, Float64,
UUID,
String> String>
null_values; null_values;
std::variant< std::variant<
@ -112,16 +116,20 @@ private:
ContainerType<UInt32>, ContainerType<UInt32>,
ContainerType<UInt64>, ContainerType<UInt64>,
ContainerType<UInt128>, ContainerType<UInt128>,
ContainerType<UInt256>,
ContainerType<Int8>, ContainerType<Int8>,
ContainerType<Int16>, ContainerType<Int16>,
ContainerType<Int32>, ContainerType<Int32>,
ContainerType<Int64>, ContainerType<Int64>,
ContainerType<Int128>,
ContainerType<Int256>,
ContainerType<Decimal32>, ContainerType<Decimal32>,
ContainerType<Decimal64>, ContainerType<Decimal64>,
ContainerType<Decimal128>, ContainerType<Decimal128>,
ContainerType<Decimal256>, ContainerType<Decimal256>,
ContainerType<Float32>, ContainerType<Float32>,
ContainerType<Float64>, ContainerType<Float64>,
ContainerType<UUID>,
ContainerType<StringRef>> ContainerType<StringRef>>
maps; maps;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;

View File

@ -58,6 +58,7 @@ namespace DB
{ {
namespace ErrorCodes namespace ErrorCodes
{ {
extern const int NOT_IMPLEMENTED;
extern const int UNSUPPORTED_METHOD; extern const int UNSUPPORTED_METHOD;
extern const int MONGODB_CANNOT_AUTHENTICATE; extern const int MONGODB_CANNOT_AUTHENTICATE;
} }
@ -189,24 +190,22 @@ BlockInputStreamPtr MongoDBDictionarySource::loadKeys(const Columns & key_column
case AttributeUnderlyingType::utUInt16: case AttributeUnderlyingType::utUInt16:
case AttributeUnderlyingType::utUInt32: case AttributeUnderlyingType::utUInt32:
case AttributeUnderlyingType::utUInt64: case AttributeUnderlyingType::utUInt64:
case AttributeUnderlyingType::utUInt128:
case AttributeUnderlyingType::utInt8: case AttributeUnderlyingType::utInt8:
case AttributeUnderlyingType::utInt16: case AttributeUnderlyingType::utInt16:
case AttributeUnderlyingType::utInt32: case AttributeUnderlyingType::utInt32:
case AttributeUnderlyingType::utInt64: case AttributeUnderlyingType::utInt64:
case AttributeUnderlyingType::utDecimal32: {
case AttributeUnderlyingType::utDecimal64:
case AttributeUnderlyingType::utDecimal128:
case AttributeUnderlyingType::utDecimal256:
key.add(attr.second.name, Int32(key_columns[attr.first]->get64(row_idx))); key.add(attr.second.name, Int32(key_columns[attr.first]->get64(row_idx)));
break; break;
}
case AttributeUnderlyingType::utFloat32: case AttributeUnderlyingType::utFloat32:
case AttributeUnderlyingType::utFloat64: case AttributeUnderlyingType::utFloat64:
{
key.add(attr.second.name, key_columns[attr.first]->getFloat64(row_idx)); key.add(attr.second.name, key_columns[attr.first]->getFloat64(row_idx));
break; break;
}
case AttributeUnderlyingType::utString: case AttributeUnderlyingType::utString:
{
String loaded_str(get<String>((*key_columns[attr.first])[row_idx])); String loaded_str(get<String>((*key_columns[attr.first])[row_idx]));
/// Convert string to ObjectID /// Convert string to ObjectID
if (attr.second.is_object_id) if (attr.second.is_object_id)
@ -219,6 +218,9 @@ BlockInputStreamPtr MongoDBDictionarySource::loadKeys(const Columns & key_column
key.add(attr.second.name, loaded_str); key.add(attr.second.name, loaded_str);
} }
break; break;
}
default:
throw Exception("Unsupported dictionary attribute type for MongoDB dictionary source", ErrorCodes::NOT_IMPLEMENTED);
} }
} }
} }

View File

@ -105,16 +105,19 @@ private:
UInt32, UInt32,
UInt64, UInt64,
UInt128, UInt128,
UInt256,
Int8, Int8,
Int16, Int16,
Int32, Int32,
Int64, Int64,
Int128,
Decimal32, Decimal32,
Decimal64, Decimal64,
Decimal128, Decimal128,
Decimal256, Decimal256,
Float32, Float32,
Float64, Float64,
UUID,
StringRef> StringRef>
null_values; null_values;
std::variant< std::variant<
@ -123,16 +126,20 @@ private:
Ptr<UInt32>, Ptr<UInt32>,
Ptr<UInt64>, Ptr<UInt64>,
Ptr<UInt128>, Ptr<UInt128>,
Ptr<UInt256>,
Ptr<Int8>, Ptr<Int8>,
Ptr<Int16>, Ptr<Int16>,
Ptr<Int32>, Ptr<Int32>,
Ptr<Int64>, Ptr<Int64>,
Ptr<Int128>,
Ptr<Int256>,
Ptr<Decimal32>, Ptr<Decimal32>,
Ptr<Decimal64>, Ptr<Decimal64>,
Ptr<Decimal128>, Ptr<Decimal128>,
Ptr<Decimal256>, Ptr<Decimal256>,
Ptr<Float32>, Ptr<Float32>,
Ptr<Float64>, Ptr<Float64>,
Ptr<UUID>,
Ptr<StringRef>> Ptr<StringRef>>
maps; maps;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;