thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10365 from ClickHouse/avoid-infinite-loop-in-hierarchical-dictionaries

Browse Source 
Avoid infinite loop in hierarchical dictionaries
This commit is contained in:
alexey-milovidov 2020-04-20 10:54:16 +03:00 committed by GitHub
commit 9d0004cba0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 153 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Core/Defines.h
View File

@ -98,3 +98,6 @@
/// Default limit on recursion depth of recursive descend parser.
#define DBMS_DEFAULT_MAX_PARSER_DEPTH 1000
/// Max depth of hierarchical dictionary
#define DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH 1000

6
src/Dictionaries/CacheDictionary.cpp
View File

@ -10,6 +10,7 @@
#include <Common/ProfilingScopedRWLock.h>
#include <Common/randomSeed.h>
#include <Common/typeid_cast.h>
#include <Core/Defines.h>
#include <ext/range.h>
#include <ext/size.h>
#include <Common/setThreadName.h>
@ -17,6 +18,7 @@
#include "DictionaryBlockInputStream.h"
#include "DictionaryFactory.h"
namespace ProfileEvents
{
extern const Event DictCacheKeysRequested;
@ -144,7 +146,7 @@ void CacheDictionary::isInImpl(const PaddedPODArray<Key> & child_ids, const Ance
PaddedPODArray<Key> children(out_size, 0);
PaddedPODArray<Key> parents(child_ids.begin(), child_ids.end());
while (true)
for (size_t i = 0; i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i)
{
size_t out_idx = 0;
size_t parents_idx = 0;
@ -218,7 +220,7 @@ void CacheDictionary::isInConstantVector(const Key child_id, const PaddedPODArra
std::vector<Key> ancestors(1, child_id);
/// Iteratively find all ancestors for child.
while (true)
for (size_t i = 0; i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i)
{
toParent(child, parent);

3
src/Dictionaries/Embedded/RegionsHierarchy.h
View File

@ -4,6 +4,7 @@
#include <boost/noncopyable.hpp>
#include <common/types.h>
#include "GeodataProviders/IHierarchiesProvider.h"
#include <Core/Defines.h>
class IRegionsHierarchyDataProvider;
@ -59,7 +60,7 @@ public:
if (lhs >= parents.size())
return false;
while (lhs != 0 && lhs != rhs)
for (size_t i = 0; lhs != 0 && lhs != rhs && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i)
lhs = parents[lhs];
return lhs != 0;

4
src/Dictionaries/FlatDictionary.cpp
View File

@ -2,6 +2,8 @@
#include <IO/WriteHelpers.h>
#include "DictionaryBlockInputStream.h"
#include "DictionaryFactory.h"
#include <Core/Defines.h>
namespace DB
{
@ -77,7 +79,7 @@ void FlatDictionary::isInImpl(const ChildType & child_ids, const AncestorType &
auto id = getAt(child_ids, row);
const auto ancestor_id = getAt(ancestor_ids, row);
while (id < loaded_size && id != null_value && id != ancestor_id)
for (size_t i = 0; id < loaded_size && id != null_value && id != ancestor_id && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i)
id = attr[id];
out[row] = id != null_value && id == ancestor_id;

4
src/Dictionaries/HashedDictionary.cpp
View File

@ -2,6 +2,8 @@
#include <ext/size.h>
#include "DictionaryBlockInputStream.h"
#include "DictionaryFactory.h"
#include <Core/Defines.h>
namespace
{
@ -87,7 +89,7 @@ void HashedDictionary::isInAttrImpl(const AttrType & attr, const ChildType & chi
auto id = getAt(child_ids, row);
const auto ancestor_id = getAt(ancestor_ids, row);
while (id != null_value && id != ancestor_id)
for (size_t i = 0; id != null_value && id != ancestor_id && i < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++i)
{
auto it = attr.find(id);
if (it != std::end(attr))

3
src/Functions/FunctionsEmbeddedDictionaries.h
View File

@ -16,6 +16,7 @@
#include <Dictionaries/Embedded/RegionsNames.h>
#include <IO/WriteHelpers.h>
#include <Common/typeid_cast.h>
#include <Core/Defines.h>
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
@ -450,7 +451,7 @@ public:
for (size_t i = 0; i < size; ++i)
{
T cur = vec_from[i];
while (cur)
for (size_t depth = 0; cur && depth < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++depth)
{
res_values.push_back(cur);
cur = Transform::toParent(cur, dict);

39
tests/queries/0_stateless/01251_dict_is_in_infinite_loop.reference Normal file
View File

@ -0,0 +1,39 @@
1
1
1
1
1
0
0
0
0
[11,22]
[22,11]
[11,22]
[22,11]
1
1
1
1
1
0
0
0
0
[11,22]
[22,11]
[11,22]
[22,11]
1
1
1
1
1
255
255
0
255
[11,22]
[22,11]
[11,22]
[22,11]

97
tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql Normal file
View File

@ -0,0 +1,97 @@
DROP DATABASE IF EXISTS database_for_dict;
CREATE DATABASE database_for_dict Engine = Ordinary;
DROP TABLE IF EXISTS database_for_dict.dict_source;
CREATE TABLE database_for_dict.dict_source (id UInt64, parent_id UInt64, value String) ENGINE = Memory;
INSERT INTO database_for_dict.dict_source VALUES (1, 0, 'hello'), (2, 1, 'world'), (3, 2, 'upyachka'), (11, 22, 'a'), (22, 11, 'b');
DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy;
CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy
(
id UInt64, parent_id UInt64 HIERARCHICAL, value String
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source'))
LAYOUT(HASHED())
LIFETIME(MIN 1 MAX 1);
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)));
DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy;
CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy
(
id UInt64, parent_id UInt64 HIERARCHICAL, value String
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source'))
LAYOUT(FLAT())
LIFETIME(MIN 1 MAX 1);
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)));
DROP DICTIONARY IF EXISTS database_for_dict.dictionary_with_hierarchy;
CREATE DICTIONARY database_for_dict.dictionary_with_hierarchy
(
id UInt64, parent_id UInt64 HIERARCHICAL, value String
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dict' table 'dict_source'))
LAYOUT(CACHE(SIZE_IN_CELLS 10))
LIFETIME(MIN 1 MAX 1);
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(2), toUInt64(1));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(11));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(22)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(11)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(22), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), toUInt64(111));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(11), materialize(toUInt64(222)));
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)), materialize(toUInt64(111)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(11));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', toUInt64(22));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(11)));
SELECT dictGetHierarchy('database_for_dict.dictionary_with_hierarchy', materialize(toUInt64(22)));
DROP DICTIONARY database_for_dict.dictionary_with_hierarchy;
DROP TABLE database_for_dict.dict_source;
DROP DATABASE database_for_dict;