thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into system.graphite_retentions

Browse Source
This commit is contained in:
Mikhail f. Shiryaev 2019-03-08 17:41:40 +01:00
commit 2d27331f0c
15 changed files with 91 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dbms/src/Common/Allocator.cpp
View File

@ -5,6 +5,7 @@
#endif
#include <cstdlib>
#include <algorithm>
#include <sys/mman.h>
#include <common/mremap.h>
@ -118,9 +119,11 @@ void * Allocator<clear_memory_>::realloc(void * buf, size_t old_size, size_t new
if (old_size == new_size)
{
/// nothing to do.
/// BTW, it's not possible to change alignment while doing realloc.
}
else if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT)
{
/// Resize malloc'd memory region with no special alignment requirement.
CurrentMemoryTracker::realloc(old_size, new_size);
void * new_buf = ::realloc(buf, new_size);
@ -133,6 +136,7 @@ void * Allocator<clear_memory_>::realloc(void * buf, size_t old_size, size_t new
}
else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD)
{
/// Resize mmap'd memory region.
CurrentMemoryTracker::realloc(old_size, new_size);
// On apple and freebsd self-implemented mremap used (common/mremap.h)
@ -142,21 +146,12 @@ void * Allocator<clear_memory_>::realloc(void * buf, size_t old_size, size_t new
/// No need for zero-fill, because mmap guarantees it.
}
else if (old_size >= MMAP_THRESHOLD && new_size < MMAP_THRESHOLD)
{
void * new_buf = alloc(new_size, alignment);
memcpy(new_buf, buf, new_size);
if (0 != munmap(buf, old_size))
{
::free(new_buf);
DB::throwFromErrno("Allocator: Cannot munmap " + formatReadableSizeWithBinarySuffix(old_size) + ".", DB::ErrorCodes::CANNOT_MUNMAP);
}
buf = new_buf;
}
else
{
/// All other cases that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods.
void * new_buf = alloc(new_size, alignment);
memcpy(new_buf, buf, old_size);
memcpy(new_buf, buf, std::min(old_size, new_size));
free(buf, old_size);
buf = new_buf;
}

22
dbms/src/Common/CompactArray.h
View File

@ -55,6 +55,28 @@ public:
return locus;
}
/// Used only in arcadia/metrika
void readText(ReadBuffer & in)
{
for (size_t i = 0; i < BITSET_SIZE; ++i)
{
if (i != 0)
assertChar(',', in);
readIntText(bitset[i], in);
}
}
/// Used only in arcadia/metrika
void writeText(WriteBuffer & out) const
{
for (size_t i = 0; i < BITSET_SIZE; ++i)
{
if (i != 0)
writeCString(",", out);
writeIntText(bitset[i], out);
}
}
private:
/// number of bytes in bitset
static constexpr size_t BITSET_SIZE = (static_cast<size_t>(bucket_count) * content_width + 7) / 8;

1
dbms/src/Common/PODArray.h
View File

@ -159,6 +159,7 @@ public:
size_t size() const { return (c_end - c_start) / ELEMENT_SIZE; }
size_t capacity() const { return (c_end_of_storage - c_start) / ELEMENT_SIZE; }
/// This method is safe to use only for information about memory usage.
size_t allocated_bytes() const { return c_end_of_storage - c_start + pad_right + pad_left; }
void clear() { c_end = c_start; }

2
dbms/src/Compression/CompressionFactory.cpp
View File

@ -85,7 +85,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const UInt8 byte_code) const
const auto family_code_and_creator = family_code_with_codec.find(byte_code);
if (family_code_and_creator == family_code_with_codec.end())
throw Exception("Unknown codec family code : " + toString(byte_code), ErrorCodes::UNKNOWN_CODEC);
throw Exception("Unknown codec family code: " + toString(byte_code), ErrorCodes::UNKNOWN_CODEC);
return family_code_and_creator->second({}, nullptr);
}

2
dbms/src/DataTypes/DataTypeString.cpp
View File

@ -132,7 +132,7 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars & data, ColumnSt
{
#ifdef __x86_64__
/// An optimistic branch in which more efficient copying is possible.
if (offset + 16 * UNROLL_TIMES <= data.allocated_bytes() && istr.position() + size + 16 * UNROLL_TIMES <= istr.buffer().end())
if (offset + 16 * UNROLL_TIMES <= data.capacity() && istr.position() + size + 16 * UNROLL_TIMES <= istr.buffer().end())
{
const __m128i * sse_src_pos = reinterpret_cast<const __m128i *>(istr.position());
const __m128i * sse_src_end = sse_src_pos + (size + (16 * UNROLL_TIMES - 1)) / 16 / UNROLL_TIMES * UNROLL_TIMES;

2
dbms/src/Interpreters/ExternalDictionaries.cpp
View File

@ -36,7 +36,7 @@ ExternalDictionaries::ExternalDictionaries(
std::move(config_repository),
&Logger::get("ExternalDictionaries"),
"external dictionary"),
context(context)
context(context)
{
init(throw_on_error);
}

41
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -155,14 +155,6 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
/// periodic update
std::vector<std::pair<std::string, LoadablePtr>> objects_to_update;
auto getNextUpdateTime = [this](const LoadablePtr & current)
{
/// calculate next update time
const auto & lifetime = current->getLifetime();
std::uniform_int_distribution<UInt64> distribution{lifetime.min_sec, lifetime.max_sec};
return std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)};
};
/// Collect objects that needs to be updated under lock. Then create new versions without lock, and assign under lock.
{
std::lock_guard lock{map_mutex};
@ -185,18 +177,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
if (!current->supportUpdates())
continue;
auto & update_time = update_times[current->getName()];
auto update_time = update_times[current->getName()];
/// check that timeout has passed
if (std::chrono::system_clock::now() < update_time)
continue;
if (!current->isModified())
{
update_time = getNextUpdateTime(current);
continue;
}
objects_to_update.emplace_back(loadable_object.first, current);
}
catch (...)
@ -209,6 +195,14 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
}
}
auto getNextUpdateTime = [this](const LoadablePtr & current)
{
/// Calculate next update time.
const auto & lifetime = current->getLifetime();
std::uniform_int_distribution<UInt64> distribution{lifetime.min_sec, lifetime.max_sec};
return std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)};
};
for (auto & [name, current] : objects_to_update)
{
LoadablePtr new_version;
@ -216,9 +210,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
try
{
/// create new version of loadable object
new_version = current->clone();
exception = new_version->getCreationException();
if (current->isModified())
{
/// Create new version of loadable object.
new_version = current->clone();
exception = new_version->getCreationException();
}
}
catch (...)
{
@ -235,8 +232,12 @@ void ExternalLoader::reloadAndUpdate(bool throw_on_error)
it->second.exception = exception;
if (!exception)
{
it->second.loadable.reset();
it->second.loadable = std::move(new_version);
/// If the dictionary is not modified - leave old version.
if (new_version)
{
it->second.loadable.reset();
it->second.loadable = std::move(new_version);
}
}
else
{

2
dbms/src/Interpreters/Set.h
View File

@ -192,4 +192,4 @@ private:
std::vector<KeyTuplePositionMapping> indexes_mapping;
};
}
}

2
dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp
View File

@ -26,7 +26,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream(
{
std::stringstream message;
message << "Reading " << data_part->marks_count << " marks from part " << data_part->name
<< ", totaly " << data_part->rows_count
<< ", total " << data_part->rows_count
<< " rows starting from the beginning of the part, columns: ";
for (size_t i = 0, size = columns_to_read.size(); i < size; ++i)
message << (i == 0 ? "" : ", ") << columns_to_read[i];

4
dbms/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql
View File

@ -26,6 +26,7 @@ CREATE TABLE test.clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('
CREATE TABLE test.clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/clear_column', '2', d, d, 8192);
INSERT INTO test.clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01');
SYSTEM SYNC REPLICA test.clear_column2;
SET replication_alter_partitions_sync=2;
ALTER TABLE test.clear_column1 ADD COLUMN s String;
@ -33,9 +34,10 @@ ALTER TABLE test.clear_column1 CLEAR COLUMN s IN PARTITION '200001';
INSERT INTO test.clear_column1 VALUES ('2000-01-01', 1, 'a'), ('2000-02-01', 2, 'b');
INSERT INTO test.clear_column1 VALUES ('2000-01-01', 3, 'c'), ('2000-02-01', 4, 'd');
SYSTEM SYNC REPLICA test.clear_column2;
SELECT 'all';
SELECT * FROM test.clear_column1 ORDER BY d, i, s;
SELECT * FROM test.clear_column2 ORDER BY d, i, s;
SELECT 'w/o i 1';
ALTER TABLE test.clear_column1 CLEAR COLUMN i IN PARTITION '200001';

3
dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.reference
View File

@ -1,2 +1 @@
0
0
killed

15
dbms/tests/queries/0_stateless/00909_kill_not_initialized_query.sh
View File

@ -36,11 +36,16 @@ sleep 1
# Kill $query_for_pending SYNC. This query is not blocker, so it should be killed fast.
timeout 5 $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null
# But let's sleep a little time, just to be sure
sleep 3
# Both queries have to be killed, doesn't matter with SYNC or ASYNC kill
$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_for_pending'"
$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_to_kill'"
for run in {1..15}
do
sleep 1
no_first_query=`$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_for_pending'"`
no_second_query=`$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes where query='$query_to_kill'"`
if [ $no_first_query == "0" ] && [ $no_second_query == "0" ]; then
echo "killed"
break
fi
done
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test.cannot_kill_query" &>/dev/null

4
dbms/tests/queries/0_stateless/00914_join_bgranvea.reference Normal file
View File

@ -0,0 +1,4 @@
b1 b1
b1 b1
b1 b1
b1 b1

15
dbms/tests/queries/0_stateless/00914_join_bgranvea.sql Normal file
View File

@ -0,0 +1,15 @@
USE test;
DROP TABLE IF EXISTS table1;
DROP TABLE IF EXISTS table2;
CREATE TABLE table1 (A String, B String, ts DateTime) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY (ts, A, B);
CREATE TABLE table2 (B String, ts DateTime) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY (ts, B);
insert into table1 values('a1','b1','2019-02-05 16:50:00'),('a1','b1','2019-02-05 16:55:00');
insert into table2 values('b1','2019-02-05 16:50:00'),('b1','2019-02-05 16:55:00');
SELECT t1.B, t2.B FROM table1 t1 ALL INNER JOIN table2 t2 ON t1.B = t2.B ORDER BY t1.B, t2.B;
DROP TABLE table1;
DROP TABLE table2;

4
libs/libdaemon/src/BaseDaemon.cpp
View File

@ -700,12 +700,12 @@ static void checkRequiredInstructions(volatile InstructionFail & fail)
#if __AVX2__
fail = InstructionFail::AVX2;
__asm__ volatile ("vpabsw %%ymm0, %%ymm0, %%ymm0" : : : "ymm0");
__asm__ volatile ("vpabsw %%ymm0, %%ymm0" : : : "ymm0");
#endif
#if __AVX512__
fail = InstructionFail::AVX512;
__asm__ volatile ("vpabsw %%zmm0, %%zmm0, %%zmm0" : : : "zmm0");
__asm__ volatile ("vpabsw %%zmm0, %%zmm0" : : : "zmm0");
#endif
fail = InstructionFail::NONE;