mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into yandex-to-clickhouse-in-configs
This commit is contained in:
mergify[bot]
GitHub
commit
64669a28ce
@ -92,7 +92,7 @@ if __name__ == "__main__":
|
||||
logging.info("Some exception occured %s", str(ex))
|
||||
raise
|
||||
finally:
|
||||
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
if os.path.exists(temp_archive_path):
|
||||
os.remove(temp_archive_path)
|
||||
logging.info("Processing of %s finished", dataset)
|
||||
|
||||
@ -92,7 +92,7 @@ if __name__ == "__main__":
|
||||
logging.info("Some exception occured %s", str(ex))
|
||||
raise
|
||||
finally:
|
||||
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
if os.path.exists(temp_archive_path):
|
||||
os.remove(temp_archive_path)
|
||||
logging.info("Processing of %s finished", dataset)
|
||||
|
||||
@ -42,7 +42,11 @@ md5sum hits_v1.tsv
|
||||
# Checksum should be equal to: f3631b6295bf06989c1437491f7592cb
|
||||
# now create table
|
||||
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets"
|
||||
# for hits_v1
|
||||
clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
|
||||
# for hits_100m_obfuscated
|
||||
clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
|
||||
|
||||
# import data
|
||||
cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000
|
||||
# optionally you can optimize table
|
||||
|
||||
@ -56,6 +56,7 @@ toc_title: Adopters
|
||||
| <a href="https://geniee.co.jp" class="favicon">Geniee</a> | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) |
|
||||
| <a href="https://www.genotek.ru/" class="favicon">Genotek</a> | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) |
|
||||
| <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
|
||||
| <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
|
||||
| <a href="https://www.huya.com/" class="favicon">HUYA</a> | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) |
|
||||
| <a href="https://www.the-ica.com/" class="favicon">ICA</a> | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) |
|
||||
| <a href="https://www.idealista.com" class="favicon">Idealista</a> | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.com/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) |
|
||||
|
||||
@ -594,14 +594,14 @@ Result:
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## h3ResIsClassIII {#h3resisclassIII}
|
||||
## h3IsResClassIII {#h3isresclassIII}
|
||||
|
||||
Returns whether [H3](#h3index) index has a resolution with Class III orientation.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
h3ResIsClassIII(index)
|
||||
h3IsResClassIII(index)
|
||||
```
|
||||
|
||||
**Parameter**
|
||||
@ -620,7 +620,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md).
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT h3ResIsClassIII(617420388352917503) as res;
|
||||
SELECT h3IsResClassIII(617420388352917503) as res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
@ -155,7 +155,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
|
||||
}
|
||||
|
||||
/// Combinators of aggregate functions.
|
||||
/// For every aggregate function 'agg' and combiner '-Comb' there is combined aggregate function with name 'aggComb',
|
||||
/// For every aggregate function 'agg' and combiner '-Comb' there is a combined aggregate function with the name 'aggComb',
|
||||
/// that can have different number and/or types of arguments, different result type and different behaviour.
|
||||
|
||||
if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name))
|
||||
@ -172,13 +172,12 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
|
||||
|
||||
String nested_name = name.substr(0, name.size() - combinator_name.size());
|
||||
/// Nested identical combinators (i.e. uniqCombinedIfIf) is not
|
||||
/// supported (since they even don't work -- silently).
|
||||
/// supported (since they don't work -- silently).
|
||||
///
|
||||
/// But non-identical does supported and works, for example
|
||||
/// uniqCombinedIfMergeIf, it is useful in case when the underlying
|
||||
/// But non-identical is supported and works. For example,
|
||||
/// uniqCombinedIfMergeIf is useful in cases when the underlying
|
||||
/// storage stores AggregateFunction(uniqCombinedIf) and in SELECT you
|
||||
/// need to filter aggregation result based on another column for
|
||||
/// example.
|
||||
/// need to filter aggregation result based on another column.
|
||||
if (!combinator->supportsNesting() && nested_name.ends_with(combinator_name))
|
||||
{
|
||||
throw Exception(ErrorCodes::ILLEGAL_AGGREGATION,
|
||||
@ -234,7 +233,7 @@ std::optional<AggregateFunctionProperties> AggregateFunctionFactory::tryGetPrope
|
||||
return found.properties;
|
||||
|
||||
/// Combinators of aggregate functions.
|
||||
/// For every aggregate function 'agg' and combiner '-Comb' there is combined aggregate function with name 'aggComb',
|
||||
/// For every aggregate function 'agg' and combiner '-Comb' there is a combined aggregate function with the name 'aggComb',
|
||||
/// that can have different number and/or types of arguments, different result type and different behaviour.
|
||||
|
||||
if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name))
|
||||
|
||||
@ -208,6 +208,12 @@ void RemoteQueryExecutor::sendQuery()
|
||||
auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings);
|
||||
ClientInfo modified_client_info = context->getClientInfo();
|
||||
modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY;
|
||||
/// Set initial_query_id to query_id for the clickhouse-benchmark.
|
||||
///
|
||||
/// (since first query of clickhouse-benchmark will be issued as SECONDARY_QUERY,
|
||||
/// due to it executes queries via RemoteBlockInputStream)
|
||||
if (modified_client_info.initial_query_id.empty())
|
||||
modified_client_info.initial_query_id = query_id;
|
||||
if (CurrentThread::isInitialized())
|
||||
{
|
||||
modified_client_info.client_trace_context = CurrentThread::get().thread_trace_context;
|
||||
|
||||
@ -24,12 +24,12 @@ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
namespace
|
||||
{
|
||||
|
||||
class FunctionH3ResIsClassIII : public IFunction
|
||||
class FunctionH3IsResClassIII : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "h3ResIsClassIII";
|
||||
static constexpr auto name = "h3IsResClassIII";
|
||||
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3ResIsClassIII>(); }
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3IsResClassIII>(); }
|
||||
|
||||
std::string getName() const override { return name; }
|
||||
|
||||
@ -69,9 +69,9 @@ public:
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionH3ResIsClassIII(FunctionFactory & factory)
|
||||
void registerFunctionH3IsResClassIII(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionH3ResIsClassIII>();
|
||||
factory.registerFunction<FunctionH3IsResClassIII>();
|
||||
}
|
||||
|
||||
}
|
||||
@ -42,7 +42,7 @@ void registerFunctionH3IndexesAreNeighbors(FunctionFactory &);
|
||||
void registerFunctionStringToH3(FunctionFactory &);
|
||||
void registerFunctionH3ToString(FunctionFactory &);
|
||||
void registerFunctionH3HexAreaM2(FunctionFactory &);
|
||||
void registerFunctionH3ResIsClassIII(FunctionFactory &);
|
||||
void registerFunctionH3IsResClassIII(FunctionFactory &);
|
||||
void registerFunctionH3IsPentagon(FunctionFactory &);
|
||||
void registerFunctionH3GetFaces(FunctionFactory &);
|
||||
#endif
|
||||
@ -98,7 +98,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
|
||||
registerFunctionStringToH3(factory);
|
||||
registerFunctionH3ToString(factory);
|
||||
registerFunctionH3HexAreaM2(factory);
|
||||
registerFunctionH3ResIsClassIII(factory);
|
||||
registerFunctionH3IsResClassIII(factory);
|
||||
registerFunctionH3IsPentagon(factory);
|
||||
registerFunctionH3GetFaces(factory);
|
||||
#endif
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
#include <rocksdb/convenience.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <shared_mutex>
|
||||
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
@ -213,9 +214,9 @@ public:
|
||||
std::vector<rocksdb::Slice> slices_keys(num_keys);
|
||||
|
||||
const auto & sample_block = metadata_snapshot->getSampleBlock();
|
||||
const auto & key_column = sample_block.getByName(storage.primary_key);
|
||||
const auto & key_column = sample_block.getByName(storage.getPrimaryKey());
|
||||
auto columns = sample_block.cloneEmptyColumns();
|
||||
size_t primary_key_pos = sample_block.getPositionByName(storage.primary_key);
|
||||
size_t primary_key_pos = sample_block.getPositionByName(storage.getPrimaryKey());
|
||||
|
||||
size_t rows_processed = 0;
|
||||
while (it < end && rows_processed < max_block_size)
|
||||
@ -230,8 +231,7 @@ public:
|
||||
}
|
||||
|
||||
std::vector<String> values;
|
||||
auto statuses = storage.rocksdb_ptr->MultiGet(rocksdb::ReadOptions(), slices_keys, &values);
|
||||
|
||||
auto statuses = storage.multiGet(slices_keys, values);
|
||||
for (size_t i = 0; i < statuses.size(); ++i)
|
||||
{
|
||||
if (statuses[i].ok())
|
||||
@ -285,7 +285,10 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_,
|
||||
|
||||
void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &)
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> lock(rocksdb_ptr_mx);
|
||||
rocksdb_ptr->Close();
|
||||
rocksdb_ptr = nullptr;
|
||||
|
||||
fs::remove_all(rocksdb_dir);
|
||||
fs::create_directories(rocksdb_dir);
|
||||
initDb();
|
||||
@ -460,9 +463,20 @@ static StoragePtr create(const StorageFactory::Arguments & args)
|
||||
|
||||
std::shared_ptr<rocksdb::Statistics> StorageEmbeddedRocksDB::getRocksDBStatistics() const
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lock(rocksdb_ptr_mx);
|
||||
if (!rocksdb_ptr)
|
||||
return nullptr;
|
||||
return rocksdb_ptr->GetOptions().statistics;
|
||||
}
|
||||
|
||||
std::vector<rocksdb::Status> StorageEmbeddedRocksDB::multiGet(const std::vector<rocksdb::Slice> & slices_keys, std::vector<String> & values) const
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lock(rocksdb_ptr_mx);
|
||||
if (!rocksdb_ptr)
|
||||
return {};
|
||||
return rocksdb_ptr->MultiGet(rocksdb::ReadOptions(), slices_keys, &values);
|
||||
}
|
||||
|
||||
void registerStorageEmbeddedRocksDB(StorageFactory & factory)
|
||||
{
|
||||
StorageFactory::StorageFeatures features{
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <shared_mutex>
|
||||
#include <common/shared_ptr_helper.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <rocksdb/status.h>
|
||||
|
||||
|
||||
namespace rocksdb
|
||||
@ -20,7 +22,6 @@ class Context;
|
||||
class StorageEmbeddedRocksDB final : public shared_ptr_helper<StorageEmbeddedRocksDB>, public IStorage, WithContext
|
||||
{
|
||||
friend struct shared_ptr_helper<StorageEmbeddedRocksDB>;
|
||||
friend class EmbeddedRocksDBSource;
|
||||
friend class EmbeddedRocksDBSink;
|
||||
friend class EmbeddedRocksDBBlockInputStream;
|
||||
public:
|
||||
@ -50,6 +51,8 @@ public:
|
||||
Strings getDataPaths() const override { return {rocksdb_dir}; }
|
||||
|
||||
std::shared_ptr<rocksdb::Statistics> getRocksDBStatistics() const;
|
||||
std::vector<rocksdb::Status> multiGet(const std::vector<rocksdb::Slice> & slices_keys, std::vector<String> & values) const;
|
||||
const String & getPrimaryKey() const { return primary_key; }
|
||||
|
||||
protected:
|
||||
StorageEmbeddedRocksDB(const StorageID & table_id_,
|
||||
@ -63,6 +66,7 @@ private:
|
||||
const String primary_key;
|
||||
using RocksDBPtr = std::unique_ptr<rocksdb::DB>;
|
||||
RocksDBPtr rocksdb_ptr;
|
||||
mutable std::shared_mutex rocksdb_ptr_mx;
|
||||
String rocksdb_dir;
|
||||
|
||||
void initDb();
|
||||
|
||||
@ -43,7 +43,8 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con
|
||||
const auto access = context->getAccess();
|
||||
const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
|
||||
|
||||
std::map<String, std::map<String, StoragePtr>> tables;
|
||||
using RocksDBStoragePtr = std::shared_ptr<StorageEmbeddedRocksDB>;
|
||||
std::map<String, std::map<String, RocksDBStoragePtr>> tables;
|
||||
for (const auto & db : DatabaseCatalog::instance().getDatabases())
|
||||
{
|
||||
const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
|
||||
@ -51,18 +52,17 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con
|
||||
for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
|
||||
{
|
||||
StoragePtr table = iterator->table();
|
||||
if (!table)
|
||||
RocksDBStoragePtr rocksdb_table = table ? std::dynamic_pointer_cast<StorageEmbeddedRocksDB>(table) : nullptr;
|
||||
if (!rocksdb_table)
|
||||
continue;
|
||||
|
||||
if (!dynamic_cast<const StorageEmbeddedRocksDB *>(table.get()))
|
||||
continue;
|
||||
if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name()))
|
||||
continue;
|
||||
tables[db.first][iterator->name()] = table;
|
||||
|
||||
tables[db.first][iterator->name()] = rocksdb_table;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
MutableColumnPtr col_database_mut = ColumnString::create();
|
||||
MutableColumnPtr col_table_mut = ColumnString::create();
|
||||
|
||||
@ -101,10 +101,9 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con
|
||||
String database = (*col_database_to_filter)[i].safeGet<const String &>();
|
||||
String table = (*col_table_to_filter)[i].safeGet<const String &>();
|
||||
|
||||
auto & rocksdb_table = dynamic_cast<StorageEmbeddedRocksDB &>(*tables[database][table]);
|
||||
auto statistics = rocksdb_table.getRocksDBStatistics();
|
||||
auto statistics = tables[database][table]->getRocksDBStatistics();
|
||||
if (!statistics)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "rocksdb statistics is not enabled");
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "RocksDB statistics are not available");
|
||||
|
||||
for (auto [tick, name] : rocksdb::TickersNameMap)
|
||||
{
|
||||
|
||||
@ -55,7 +55,7 @@ export -f insert1
|
||||
export -f insert2
|
||||
export -f select1
|
||||
export -f select2
|
||||
export -f truncate1
|
||||
# export -f truncate1
|
||||
|
||||
for _ in {1..5}; do
|
||||
timeout $TIMEOUT bash -c insert1 &
|
||||
@ -64,7 +64,7 @@ done
|
||||
|
||||
timeout $TIMEOUT bash -c select1 &
|
||||
timeout $TIMEOUT bash -c select2 &
|
||||
timeout $TIMEOUT bash -c truncate1 &
|
||||
# timeout $TIMEOUT bash -c truncate1 &
|
||||
|
||||
wait
|
||||
echo "OK"
|
||||
|
||||
49
tests/queries/0_stateless/02030_rocksdb_race_long.sh
Executable file
49
tests/queries/0_stateless/02030_rocksdb_race_long.sh
Executable file
@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: race
|
||||
|
||||
unset CLICKHOUSE_LOG_COMMENT
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
echo "
|
||||
DROP TABLE IF EXISTS rocksdb_race;
|
||||
CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key);
|
||||
INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000);
|
||||
" | $CLICKHOUSE_CLIENT -n
|
||||
|
||||
function read_stat_thread()
|
||||
{
|
||||
while true; do
|
||||
echo "
|
||||
SELECT * FROM system.rocksdb FORMAT Null;
|
||||
" | $CLICKHOUSE_CLIENT -n
|
||||
done
|
||||
}
|
||||
|
||||
function truncate_thread()
|
||||
{
|
||||
while true; do
|
||||
sleep 3s;
|
||||
echo "
|
||||
TRUNCATE TABLE rocksdb_race;
|
||||
" | $CLICKHOUSE_CLIENT -n
|
||||
done
|
||||
}
|
||||
|
||||
# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout
|
||||
export -f read_stat_thread;
|
||||
export -f truncate_thread;
|
||||
|
||||
TIMEOUT=20
|
||||
|
||||
timeout $TIMEOUT bash -c read_stat_thread 2> /dev/null &
|
||||
timeout $TIMEOUT bash -c truncate_thread 2> /dev/null &
|
||||
|
||||
wait
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE rocksdb_race"
|
||||
@ -0,0 +1 @@
|
||||
3
|
||||
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
query_id="${CLICKHOUSE_DATABASE}_$$"
|
||||
benchmark_args=(
|
||||
--iterations 1
|
||||
--log_queries 1
|
||||
--query_id "$query_id"
|
||||
--log_queries_min_type QUERY_FINISH
|
||||
)
|
||||
$CLICKHOUSE_BENCHMARK "${benchmark_args[@]}" --query "SELECT * FROM remote('127.2', 'system.one')" >& /dev/null
|
||||
$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
|
||||
# Number of queries:
|
||||
# - DESC TABLE system.one
|
||||
# - query on initiator
|
||||
# - query on shard
|
||||
# Total: 3
|
||||
#
|
||||
# -- NOTE: this test cannot use 'current_database = $CLICKHOUSE_DATABASE',
|
||||
# -- because it does not propagated via remote queries,
|
||||
# -- but it uses query_id, and this is enough.
|
||||
$CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "SELECT count() FROM system.query_log WHERE event_date >= yesterday() AND initial_query_id = {query_id:String}"
|
||||
@ -11,6 +11,6 @@ INSERT INTO h3_indexes VALUES (geoToH3(10.0, 0.0, 1));
|
||||
INSERT INTO h3_indexes VALUES (geoToH3(0.0, 10.0, 2));
|
||||
INSERT INTO h3_indexes VALUES (geoToH3(10.0, 10.0, 3));
|
||||
|
||||
SELECT h3ResIsClassIII(h3_index) FROM h3_indexes ORDER BY h3_index;
|
||||
SELECT h3IsResClassIII(h3_index) FROM h3_indexes ORDER BY h3_index;
|
||||
|
||||
DROP TABLE h3_indexes;
|
||||
@ -68,7 +68,7 @@ int main(int argc, char *argv[])
|
||||
|
||||
LOG_INFO(logger, "Last committed index: {}", last_commited_index);
|
||||
|
||||
DB::KeeperLogStore changelog(argv[2], 10000000, true);
|
||||
DB::KeeperLogStore changelog(argv[2], 10000000, true, settings->compress_logs);
|
||||
changelog.init(last_commited_index, 10000000000UL); /// collect all logs
|
||||
if (changelog.size() == 0)
|
||||
LOG_INFO(logger, "Changelog empty");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user