diff --git a/README.md b/README.md index fa8df29468e..3b5209dcbe9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ curl https://clickhouse.com/ | sh Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know. -* [v24.9 Community Call](https://clickhouse.com/company/events/v24-9-community-release-call) - September 26 +* [v24.10 Community Call](https://clickhouse.com/company/events/v24-10-community-release-call) - October 31 ## Upcoming Events diff --git a/ci_v2/docker/style-test/Dockerfile b/ci_v2/docker/style-test/Dockerfile new file mode 100644 index 00000000000..165cdc3dcb1 --- /dev/null +++ b/ci_v2/docker/style-test/Dockerfile @@ -0,0 +1,17 @@ +# docker build -t clickhouse/style-test . +FROM ubuntu:22.04 + +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + aspell \ + libxml2-utils \ + python3-pip \ + locales \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + +COPY requirements.txt / +RUN pip3 install --no-cache-dir -r requirements.txt diff --git a/ci_v2/docker/style-test/requirements.txt b/ci_v2/docker/style-test/requirements.txt new file mode 100644 index 00000000000..987b014d9ba --- /dev/null +++ b/ci_v2/docker/style-test/requirements.txt @@ -0,0 +1,4 @@ +requests==2.32.3 +yamllint==1.26.3 +codespell==2.2.1 +https://clickhouse-builds.s3.amazonaws.com/packages/praktika-0.1-py3-none-any.whl diff --git a/ci_v2/jobs/check_style.py b/ci_v2/jobs/check_style.py new file mode 100644 index 00000000000..4dd3864e865 --- /dev/null +++ b/ci_v2/jobs/check_style.py @@ -0,0 +1,410 @@ +import math +import multiprocessing +import os +import re +import sys +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +from praktika.result import Result +from praktika.utils import Shell, Utils + +NPROC = multiprocessing.cpu_count() + + +def chunk_list(data, n): + """Split the data list into n nearly equal-sized chunks.""" + chunk_size = math.ceil(len(data) / n) + for i in range(0, len(data), chunk_size): + yield data[i : i + chunk_size] + + +def run_check_concurrent(check_name, check_function, files, nproc=NPROC): + stop_watch = Utils.Stopwatch() + + if not files: + print(f"File list is empty [{files}]") + raise + + file_chunks = list(chunk_list(files, nproc)) + results = [] + + # Run check_function concurrently on each chunk + with ProcessPoolExecutor(max_workers=NPROC) as executor: + futures = [executor.submit(check_function, chunk) for chunk in file_chunks] + # Wait for results and process them (optional) + for future in futures: + try: + res = future.result() + if res and res not in results: + results.append(res) + except Exception as e: + results.append(f"Exception in {check_name}: {e}") + + result = Result( + name=check_name, + status=Result.Status.SUCCESS if not results else Result.Status.FAILED, + start_time=stop_watch.start_time, + duration=stop_watch.duration, + info=f"errors: {results}" if results else "", + ) + return result + + +def run_simple_check(check_name, check_function, **kwargs): + stop_watch = Utils.Stopwatch() + + error = check_function(**kwargs) + + result = Result( + name=check_name, + status=Result.Status.SUCCESS if not error else Result.Status.FAILED, + start_time=stop_watch.start_time, + duration=stop_watch.duration, + info=error, + ) + return result + + +def run_check(check_name, check_function, files): + return run_check_concurrent(check_name, check_function, files, nproc=1) + + +def check_duplicate_includes(file_path): + includes = [] + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + for line in f: + if re.match(r"^#include ", line): + includes.append(line.strip()) + + include_counts = {line: includes.count(line) for line in includes} + duplicates = {line: count for line, count in include_counts.items() if count > 1} + + if duplicates: + return f"{file_path}: {duplicates}" + return "" + + +def check_whitespaces(file_paths): + for file in file_paths: + exit_code, out, err = Shell.get_res_stdout_stderr( + f'./ci_v2/jobs/scripts/check_style/double_whitespaces.pl "{file}"', + verbose=False, + ) + if out or err: + return out + " err: " + err + return "" + + +def check_yamllint(file_paths): + file_paths = " ".join([f"'{file}'" for file in file_paths]) + exit_code, out, err = Shell.get_res_stdout_stderr( + f"yamllint --config-file=./.yamllint {file_paths}", verbose=False + ) + return out or err + + +def check_xmllint(file_paths): + if not isinstance(file_paths, list): + file_paths = [file_paths] + file_paths = " ".join([f"'{file}'" for file in file_paths]) + exit_code, out, err = Shell.get_res_stdout_stderr( + f"xmllint --noout --nonet {file_paths}", verbose=False + ) + return out or err + + +def check_functional_test_cases(files): + """ + Queries with event_date should have yesterday() not today() + NOTE: it is not that accuate, but at least something. + """ + + patterns = [ + re.compile( + r"(?i)where.*?\bevent_date\s*(=|>=)\s*today\(\)(?!\s*-\s*1)", + re.IGNORECASE | re.DOTALL, + ) + ] + + errors = [] + for test_case in files: + try: + with open(test_case, "r", encoding="utf-8", errors="replace") as f: + file_content = " ".join( + f.read().splitlines() + ) # Combine lines into a single string + + # Check if any pattern matches in the concatenated string + if any(pattern.search(file_content) for pattern in patterns): + errors.append( + f"event_date should be filtered using >=yesterday() in {test_case} (to avoid flakiness)" + ) + + except Exception as e: + errors.append(f"Error checking {test_case}: {e}") + + for test_case in files: + if "fail" in test_case: + errors.append(f"test case {test_case} includes 'fail' in its name") + + return " ".join(errors) + + +def check_gaps_in_tests_numbers(file_paths, gap_threshold=100): + test_numbers = set() + + pattern = re.compile(r"(\d+)") + + for file in file_paths: + file_name = os.path.basename(file) + match = pattern.search(file_name) + if match: + test_numbers.add(int(match.group(1))) + + sorted_numbers = sorted(test_numbers) + large_gaps = [] + for i in range(1, len(sorted_numbers)): + prev_num = sorted_numbers[i - 1] + next_num = sorted_numbers[i] + diff = next_num - prev_num + if diff >= gap_threshold: + large_gaps.append(f"Gap ({prev_num}, {next_num}) > {gap_threshold}") + + return large_gaps + + +def check_broken_links(path, exclude_paths): + broken_symlinks = [] + + for path in Path(path).rglob("*"): + if any(exclude_path in str(path) for exclude_path in exclude_paths): + continue + if path.is_symlink(): + if not path.exists(): + broken_symlinks.append(str(path)) + + if broken_symlinks: + for symlink in broken_symlinks: + print(symlink) + return f"Broken symlinks found: {broken_symlinks}" + else: + return "" + + +def check_cpp_code(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check_cpp.sh" + ) + if err: + out += err + return out + + +def check_repo_submodules(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check_submodules.sh" + ) + if err: + out += err + return out + + +def check_other(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/checks_to_refactor.sh" + ) + if err: + out += err + return out + + +def check_codespell(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check_typos.sh" + ) + if err: + out += err + return out + + +def check_aspell(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check_aspell.sh" + ) + if err: + out += err + return out + + +def check_mypy(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check-mypy" + ) + if err: + out += err + return out + + +def check_pylint(): + res, out, err = Shell.get_res_stdout_stderr( + "./ci_v2/jobs/scripts/check_style/check-pylint" + ) + if err: + out += err + return out + + +def check_file_names(files): + files_set = set() + for file in files: + file_ = file.lower() + if file_ in files_set: + return f"Non-uniq file name in lower case: {file}" + files_set.add(file_) + return "" + + +if __name__ == "__main__": + results = [] + stop_watch = Utils.Stopwatch() + + all_files = Utils.traverse_paths( + include_paths=["."], + exclude_paths=[ + "./.git", + "./contrib", + "./build", + ], + not_exists_ok=True, # ./build may exist if runs locally + ) + + cpp_files = Utils.traverse_paths( + include_paths=["./src", "./base", "./programs", "./utils"], + exclude_paths=[ + "./base/glibc-compatibility", + "./contrib/consistent-hashing", + "./base/widechar_width", + ], + file_suffixes=[".h", ".cpp"], + ) + + yaml_workflow_files = Utils.traverse_paths( + include_paths=["./.github"], + exclude_paths=[], + file_suffixes=[".yaml", ".yml"], + ) + + xml_files = Utils.traverse_paths( + include_paths=["."], + exclude_paths=["./.git", "./contrib/"], + file_suffixes=[".xml"], + ) + + functional_test_files = Utils.traverse_paths( + include_paths=["./tests/queries"], + exclude_paths=[], + file_suffixes=[".sql", ".sh", ".py", ".j2"], + ) + + results.append( + Result( + name="Read Files", + status=Result.Status.SUCCESS, + start_time=stop_watch.start_time, + duration=stop_watch.duration, + ) + ) + + results.append( + run_check_concurrent( + check_name="Whitespace Check", + check_function=check_whitespaces, + files=cpp_files, + ) + ) + results.append( + run_check_concurrent( + check_name="YamlLint Check", + check_function=check_yamllint, + files=yaml_workflow_files, + ) + ) + results.append( + run_check_concurrent( + check_name="XmlLint Check", + check_function=check_xmllint, + files=xml_files, + ) + ) + results.append( + run_check_concurrent( + check_name="Functional Tests scripts smoke check", + check_function=check_functional_test_cases, + files=functional_test_files, + ) + ) + results.append( + run_check( + check_name="Check Tests Numbers", + check_function=check_gaps_in_tests_numbers, + files=functional_test_files, + ) + ) + results.append( + run_simple_check( + check_name="Check Broken Symlinks", + check_function=check_broken_links, + path="./", + exclude_paths=["contrib/", "metadata/", "programs/server/data"], + ) + ) + results.append( + run_simple_check( + check_name="Check CPP code", + check_function=check_cpp_code, + ) + ) + results.append( + run_simple_check( + check_name="Check Submodules", + check_function=check_repo_submodules, + ) + ) + results.append( + run_check( + check_name="Check File Names", + check_function=check_file_names, + files=all_files, + ) + ) + results.append( + run_simple_check( + check_name="Check Many Different Things", + check_function=check_other, + ) + ) + results.append( + run_simple_check( + check_name="Check Codespell", + check_function=check_codespell, + ) + ) + results.append( + run_simple_check( + check_name="Check Aspell", + check_function=check_aspell, + ) + ) + + res = Result.create_from(results=results, stopwatch=stop_watch).dump() + + if not res.is_ok(): + print("Style check: failed") + for result in results: + if not result.is_ok(): + print("Failed check:") + print(" | ", result) + sys.exit(1) + else: + print("Style check: ok") diff --git a/ci_v2/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt b/ci_v2/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt new file mode 100644 index 00000000000..8ec2e001a73 --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt @@ -0,0 +1,3050 @@ +personal_ws-1.1 en 2984 +AArch +ACLs +ALTERs +AMPLab +AMQP +ANNIndex +ANNIndexes +ANOVA +AORM +APIs +ARMv +ASLR +ASOF +ASan +AWND +AWST +Actian +ActionsMenu +ActiveRecord +AddressSanitizer +AggregateFunction +Aggregatefunction +AggregatingMergeTree +AggregatorThreads +AggregatorThreadsActive +Akka +AlertManager +Alexey +AnyEvent +AppleClang +Approximative +ArrayJoin +ArrowStream +AsyncInsertCacheSize +AsynchronousHeavyMetricsCalculationTimeSpent +AsynchronousHeavyMetricsUpdateInterval +AsynchronousInsert +AsynchronousInsertThreads +AsynchronousInsertThreadsActive +AsynchronousMetricsCalculationTimeSpent +AsynchronousMetricsUpdateInterval +AsynchronousReadWait +Authenticator +Authenticators +AutoFDO +AutoML +Autocompletion +AvroConfluent +AzureQueue +BIGINT +BIGSERIAL +BORO +BSON +BSONEachRow +BackgroundBufferFlushSchedulePool +BackgroundBufferFlushSchedulePoolSize +BackgroundBufferFlushSchedulePoolTask +BackgroundCommonPoolSize +BackgroundCommonPoolTask +BackgroundDistributedSchedulePool +BackgroundDistributedSchedulePoolSize +BackgroundDistributedSchedulePoolTask +BackgroundFetchesPoolSize +BackgroundFetchesPoolTask +BackgroundMergesAndMutationsPoolSize +BackgroundMergesAndMutationsPoolTask +BackgroundMessageBrokerSchedulePoolSize +BackgroundMessageBrokerSchedulePoolTask +BackgroundMovePoolSize +BackgroundMovePoolTask +BackgroundProcessingPool +BackgroundSchedulePool +BackgroundSchedulePoolSize +BackgroundSchedulePoolTask +BackupsIO +BackupsIOThreads +BackupsIOThreadsActive +BackupsThreads +BackupsThreadsActive +BestEffort +BestEffortOrNull +BestEffortOrZero +BestEffortUS +BestEffortUSOrNull +BestEffortUSOrZero +Blazingly +BlockActiveTime +BlockDiscardBytes +BlockDiscardMerges +BlockDiscardOps +BlockDiscardTime +BlockInFlightOps +BlockQueueTime +BlockReadBytes +BlockReadMerges +BlockReadOps +BlockReadTime +BlockWriteBytes +BlockWriteMerges +BlockWriteOps +BlockWriteTime +Boncz +Bool +BrokenDistributedFilesToInsert +Bugfix +BuildID +BuilderBinAarch +BuilderBinAmd +Bytebase +CCTOOLS +CDATA +CDFs +CDMA +CESU +CIDR +CIDRToRange +CKMAN +CKibana +CLOB +CLion +CMPLNT +CMake +CMakeLists +CODECS +COVID +CPUFrequencyMHz +CPUs +CSVWithNames +CSVWithNamesAndTypes +CSVs +CTEs +CacheDetachedFileSegments +CacheDictionaries +CacheDictionary +CacheDictionaryThreads +CacheDictionaryThreadsActive +CacheDictionaryUpdateQueueBatches +CacheDictionaryUpdateQueueKeys +CacheFileSegments +CamelCase +Cap'n +CapContains +CapUnion +CapnProto +CatBoost +CellAreaM +CellAreaRads +CellsIntersect +CentOS +CertificateHandler +Chadmin +ChannelID +Cidr +Ciphertext +CityHash +Clangd +ClickBench +ClickCat +ClickHouse +ClickHouse's +ClickHouseClient +ClickHouseMigrator +ClickHouseNIO +ClickHouseVapor +ClickVisual +ClickableSquare +CloudAvailableBadge +CloudDetails +CloudNotSupportedBadge +CloudStorage +CodeBlock +CodeLLDB +Codecs +CollapsingMergeTree +Combinators +Compat +CompiledExpressionCacheBytes +CompiledExpressionCacheCount +ComplexKeyCache +ComplexKeyDirect +ComplexKeyHashed +Composable +Config +ConnectionDetails +Const +ContextLockWait +Contrib +CountMin +Covid +Cramer's +Criteo +Crotty +Crowdsourced +Ctrl +CurrentMetrics +CustomSeparated +CustomSeparatedWithNames +CustomSeparatedWithNamesAndTypes +DBAs +DBMSs +DBeaver +DD +DDLWORKER +DDLWorker +DDLWorkerThreads +DDLWorkerThreadsActive +DECRYPT +DELETEs +DESC +DIEs +DOGEFI +Damerau +DataGrip +DataLens +DataTime +DataTypes +DatabaseCatalog +DatabaseCatalogThreads +DatabaseCatalogThreadsActive +DatabaseOnDisk +DatabaseOnDiskThreads +DatabaseOnDiskThreadsActive +DatabaseOrdinaryThreads +DatabaseOrdinaryThreadsActive +DateTime +DateTimes +DbCL +Decrypted +Deduplicate +Deduplication +DelayedInserts +DeliveryTag +DeltaLake +Denormalize +DestroyAggregatesThreads +DestroyAggregatesThreadsActive +DictCacheRequests +DiskAvailable +DiskObjectStorage +DiskObjectStorageAsyncThreads +DiskObjectStorageAsyncThreadsActive +DiskSpaceReservedForMerge +DiskTotal +DiskUnreserved +DiskUsed +DistributedFilesToInsert +DistributedSend +DockerHub +DoubleDelta +Doxygen +Dresseler +Durre +ECMA +ETag +Ecto +EdgeAngle +EdgeLengthKm +EdgeLengthM +ElasticSearch +EmbeddedRocksDB +Embeddings +Encodings +Enum +Enums +Eoan +EphemeralNode +Ethereum +ExactEdgeLengthKm +ExactEdgeLengthM +ExactEdgeLengthRads +ExecutablePool +ExtType +ExternalDistributed +FFFD +FFFFFFFF +FIPS +FOSDEM +FQDN +Failover +FarmHash +FileCluster +FileLog +FilesystemCacheBytes +FilesystemCacheElements +FilesystemCacheFiles +FilesystemCacheReadBuffers +FilesystemCacheSize +FilesystemLogsPathAvailableBytes +FilesystemLogsPathAvailableINodes +FilesystemLogsPathTotalBytes +FilesystemLogsPathTotalINodes +FilesystemLogsPathUsedBytes +FilesystemLogsPathUsedINodes +FilesystemMainPathAvailableBytes +FilesystemMainPathAvailableINodes +FilesystemMainPathTotalBytes +FilesystemMainPathTotalINodes +FilesystemMainPathUsedBytes +FilesystemMainPathUsedINodes +FixedString +FlameGraph +Flink +ForEach +FreeBSD +Fuzzer +Fuzzers +GHCN +GTID +GTest +Gb +Gbit +Gcc +GenerateRandom +GeoCoord +Geobases +Geohash +Geoid +GetBaseCell +GetDestinationIndexFromUnidirectionalEdge +GetFaces +GetIndexesFromUnidirectionalEdge +GetNeighbors +GetOriginIndexFromUnidirectionalEdge +GetPentagonIndexes +GetRes +GetResolution +GetUnidirectionalEdge +GetUnidirectionalEdgeBoundary +GetUnidirectionalEdgesFromHexagon +GitLab +GlobalThread +GlobalThreadActive +GoLand +GoogleTest +Grafana +GraphQL +GraphiteMergeTree +Greenwald +HDDs +HHMM +HMAC +HNSW +HSTS +HTTPConnection +HTTPThreads +HashedDictionary +HashedDictionaryThreads +HashedDictionaryThreadsActive +Haversine +Heredoc +HexAreaKm +HexAreaM +HexRing +Holistics +Homebrew +Homebrew's +HorizontalDivide +Hostname +HouseOps +Hudi +HyperLogLog +Hypot +IANA +IDE +IDEs +IDNA +IMDS +INFILE +INSERTed +INSERTs +IOPrefetchThreads +IOPrefetchThreadsActive +IOThreads +IOThreadsActive +IOUringInFlightEvents +IOUringPendingEvents +IOWriterThreads +IOWriterThreadsActive +IPTrie +IProcessor +IPv +Identifiant +Incrementing +IndexesAreNeighbors +InfluxDB +Instana +IntN +Integrations +IntelliJ +IntelliSense +InterserverConnection +InterserverThreads +IntervalDay +IntervalHour +IntervalMicrosecond +IntervalMillisecond +IntervalMilliseconds +IntervalMinute +IntervalMonth +IntervalNanosecond +IntervalQuarter +IntervalSecond +IntervalWeek +IntervalYear +IsPentagon +IsResClassIII +IsValid +JBOD +JOINed +JOINs +JSONAllPaths +JSONAllPathsWithTypes +JSONArrayLength +JSONAsObject +JSONAsString +JSONColumns +JSONColumnsWithMetadata +JSONCompact +JSONCompactColumns +JSONCompactEachRow +JSONCompactEachRowWithNames +JSONCompactEachRowWithNamesAndTypes +JSONCompactStrings +JSONCompactStringsEachRow +JSONCompactStringsEachRowWithNames +JSONCompactStringsEachRowWithNamesAndTypes +JSONCompactWithProgress +JSONDynamicPaths +JSONDynamicPathsWithTypes +JSONEachRow +JSONEachRowWithProgress +JSONExtract +JSONExtractArrayRaw +JSONExtractBool +JSONExtractFloat +JSONExtractInt +JSONExtractKeys +JSONExtractKeysAndValues +JSONExtractKeysAndValuesRaw +JSONExtractRaw +JSONExtractString +JSONExtractUInt +JSONHas +JSONLength +JSONObjectEachRow +JSONSharedDataPaths +JSONSharedDataPathsWithTypes +JSONStrings +JSONStringsEachRow +JSONStringsEachRowWithProgress +JSONType +JSONs +Jaeger +Jannis +Jaro +JavaHash +Jemalloc +Jepsen +JetBrains +Jitter +Joda +JumpConsistentHash +Jupyter +KDevelop +KafkaAssignedPartitions +KafkaBackgroundReads +KafkaConsumers +KafkaConsumersInUse +KafkaConsumersWithAssignment +KafkaLibrdkafkaThreads +KafkaProducers +KafkaWrites +Kahan +Kaser +KeeperAliveConnections +KeeperMap +KeeperOutstandingRequests +Kerberos +Khanna +Kibana +KittenHouse +Klickhouse +Kolmogorov +Konstantin +Korzeniewski +Kubernetes +LDAP +LGPL +LIMITs +LINEITEM +LLDB +LLVM's +LOCALTIME +LOCALTIMESTAMP +LONGLONG +LOONGARCH +Lemire +Levenshtein +Liao +LibFuzzer +LightHouse +LineAsString +LineString +Linf +LinfDistance +LinfNorm +LinfNormalize +LinksDeployment +Linq +LoadAverage +LocalThread +LocalThreadActive +LogQL +Logstash +LookML +LoongArch +LowCardinality +LpDistance +LpNorm +LpNormalize +Luebbe +Lyft +MACNumToString +MACStringToNum +MACStringToOUI +MEDIUMINT +MEMTABLE +MMapCacheCells +MMappedAllocBytes +MMappedAllocs +MMappedFileBytes +MMappedFiles +MSSQL +MSan +MVCC +MacBook +MacOS +MapState +MarkCacheBytes +MarkCacheFiles +MarksLoaderThreads +MarksLoaderThreadsActive +MaterializedMySQL +MaterializedPostgreSQL +MaterializedView +MaxDDLEntryID +MaxMind +MaxPartCountForPartition +MaxPushedDDLEntryID +Mbps +McNeal +Memcheck +MemoryCode +MemoryDataAndStack +MemoryResident +MemoryResidentMax +MemorySanitizer +MemoryShared +MemoryTracking +MemoryVirtual +Menne +MergeJoin +MergeState +MergeTree +MergeTreeAllRangesAnnouncementsSent +MergeTreeBackgroundExecutor +MergeTreeBackgroundExecutorThreads +MergeTreeBackgroundExecutorThreadsActive +MergeTreeDataSelectExecutor +MergeTreeDataSelectExecutorThreads +MergeTreeDataSelectExecutorThreadsActive +MergeTreePartsCleanerThreads +MergeTreePartsCleanerThreadsActive +MergeTreePartsLoaderThreads +MergeTreePartsLoaderThreadsActive +MergeTreeReadTaskRequestsSent +MergeTreeSettings +MessagePack +Metastore +MetroHash +MiB +Milli +Milovidov +MinHash +MinIO +MinMax +MindsDB +Mongo +Mongodb +Monotonicity +MsgPack +MultiLineString +MultiPolygon +Multiline +Multiqueries +Multithreading +Multiword +MurmurHash +MySQLConnection +MySQLDump +MySQLThreads +NATS +NCHAR +NDJSON +NEKUDOTAYIM +NEWDATE +NEWDECIMAL +NFKC +NFKD +NOAA +NULLIF +NVME +NVMe +NYPD +NaNs +Nagios +Nambiar +Namenode +NamesAndTypesList +Nano +Nesterov +NetworkReceive +NetworkReceiveBytes +NetworkReceiveDrop +NetworkReceiveErrors +NetworkReceivePackets +NetworkSend +NetworkSendBytes +NetworkSendDrop +NetworkSendErrors +NetworkSendPackets +Noaa +NodeJs +NonMonotonic +NuRaft +NumHexagons +NumPy +NumToString +NumToStringClassC +NumberOfDatabases +NumberOfDetachedByUserParts +NumberOfDetachedParts +NumberOfTables +OFNS +OLAP +OLTP +OSContextSwitches +OSGuestNiceTime +OSGuestNiceTimeCPU +OSGuestNiceTimeNormalized +OSGuestTime +OSGuestTimeCPU +OSGuestTimeNormalized +OSIOWaitMicroseconds +OSIOWaitTime +OSIOWaitTimeCPU +OSIOWaitTimeNormalized +OSIdleTime +OSIdleTimeCPU +OSIdleTimeNormalized +OSInterrupts +OSIrqTime +OSIrqTimeCPU +OSIrqTimeNormalized +OSMemoryAvailable +OSMemoryBuffers +OSMemoryCached +OSMemoryFreePlusCached +OSMemoryFreeWithoutCached +OSMemoryTotal +OSNiceTime +OSNiceTimeCPU +OSNiceTimeNormalized +OSOpenFiles +OSProcessesBlocked +OSProcessesCreated +OSProcessesRunning +OSSoftIrqTime +OSSoftIrqTimeCPU +OSSoftIrqTimeNormalized +OSStealTime +OSStealTimeCPU +OSStealTimeNormalized +OSSystemTime +OSSystemTimeCPU +OSSystemTimeNormalized +OSThreadsRunnable +OSThreadsTotal +OSUptime +OSUserTime +OSUserTimeCPU +OSUserTimeNormalized +OTLP +OUTFILE +ObjectId +Oblakov +Observability +Octonica +Ok +OnTime +OpenCelliD +OpenFileForRead +OpenFileForWrite +OpenSSL +OpenSUSE +OpenSky +OpenStack +OpenTelemetry +OrDefault +OrNull +OrZero +OvercommitTracker +PAAMAYIM +PCRE +PRCP +PREWHERE +PROCESSLIST +PROXYv +PSUN +PagerDuty +ParallelFormattingOutputFormatThreads +ParallelFormattingOutputFormatThreadsActive +ParallelParsingInputFormat +ParallelParsingInputFormatThreads +ParallelParsingInputFormatThreadsActive +Parametrized +ParquetMetadata +Parsers +PartMutation +Partitioner +PartsActive +PartsCommitted +PartsCompact +PartsDeleteOnDestroy +PartsDeleting +PartsOutdated +PartsPreActive +PartsPreCommitted +PartsTemporary +PartsWide +PeerDB +PendingAsyncInsert +Percona +PhpStorm +PlantUML +Poess +PointDistKm +PointDistM +PointDistRads +PostHistory +PostLink +PostgreSQLConnection +PostgreSQLThreads +Postgres +PostgresSQL +Precompiled +Preprocess +PrettyCompact +PrettyCompactMonoBlock +PrettyCompactNoEscapes +PrettyCompactNoEscapesMonoBlock +PrettyJSONEachRow +PrettyMonoBlock +PrettyNoEscapes +PrettyNoEscapesMonoBlock +PrettySpace +PrettySpaceMonoBlock +PrettySpaceNoEscapes +PrettySpaceNoEscapesMonoBlock +Prewhere +PrivateKeyPassphraseHandler +ProfileEvents +Profiler +Proleptic +PromHouse +PromQL +Promql +Promtail +Protobuf +ProtobufList +ProtobufSingle +ProxySQL +Punycode +PyArrow +PyCharm +QATlib +QEMU +QTCreator +Quantile +QueryCacheBytes +QueryCacheEntries +QueryCacheHits +QueryCacheMisses +QueryPreempted +QueryThread +QuickAssist +QuickSight +QuoteMeta +RBAC +RClickHouse +RHEL +ROLLUP +RWLock +RWLockActiveReaders +RWLockActiveWriters +RWLockWaitingReaders +RWLockWaitingWriters +RabbitMQ +Rabl +RangeHashed +RawBLOB +ReDoS +ReadTaskRequestsSent +ReadonlyReplica +RecipeNLG +Recompressing +Recompression +RectAdd +RectContains +RectIntersection +RectUnion +RedHat +Redash +Reddit +Refactorings +ReferenceKeyed +Refreshable +RegexpTree +RemoteRead +ReplacingMergeTree +ReplicasMaxAbsoluteDelay +ReplicasMaxInsertsInQueue +ReplicasMaxMergesInQueue +ReplicasMaxQueueSize +ReplicasMaxRelativeDelay +ReplicasSumInsertsInQueue +ReplicasSumMergesInQueue +ReplicasSumQueueSize +ReplicatedAggregatingMergeTree +ReplicatedChecks +ReplicatedCollapsingMergeTree +ReplicatedFetch +ReplicatedGraphiteMergeTree +ReplicatedMergeTree +ReplicatedReplacingMergeTree +ReplicatedSend +ReplicatedSummingMergeTree +ReplicatedVersionedCollapsingMergeTree +Resample +RestartReplicaThreads +RestartReplicaThreadsActive +RestoreThreads +RestoreThreadsActive +RoaringBitmap +RocksDB +Rollup +RowBinary +RowBinaryWithDefaults +RowBinaryWithNames +RowBinaryWithNamesAndTypes +Runtime +SATA +SELECTs +SERIALIZABLE +SIGTERM +SIMD +SLES +SLRU +SMALLINT +SNWD +SPNEGO +SQEs +SQLAlchemy +SQLConsoleDetail +SQLInsert +SQLSTATE +SSDCache +SSDComplexKeyCache +SSDs +SSLManager +SSRF +SSSE +SaaS +Sanjeev +Sankey +Scalable +Scatterplot +Schaefer +Schemas +Schwartzian +SeasClick +SeekTable +SelfManaged +Sematext +SendExternalTables +SendScalars +ShareAlike +SharedMergeTree +Shortkeys +SimHash +Simhash +SimpleAggregateFunction +SimpleState +SipHash +Smirnov's +Smirnov'test +Soundex +SpanKind +Spearman's +SquaredDistance +SquaredNorm +StartTLS +StartTime +StartupSystemTables +StartupSystemTablesThreads +StartupSystemTablesThreadsActive +Stateful +StorageBufferBytes +StorageBufferRows +StorageDistributed +StorageDistributedThreads +StorageDistributedThreadsActive +StorageHive +StorageHiveThreads +StorageHiveThreadsActive +StorageODBC +StorageS +StringToNum +StringToNumOrDefault +StringToNumOrNull +StripeLog +Stripelog +Strohmeier +Subcolumns +Subexpression +Submodules +Subqueries +Substrings +SummingMergeTree +SuperSet +Superset +SupersetDocker +SystemReplicasThreads +SystemReplicasThreadsActive +TABLUM +TAVG +TCPConnection +TCPThreads +TDigest +TINYINT +TLSv +TMAX +TMIN +TPCH +TSDB +TSVRaw +TSVWithNames +TSVs +TSan +TThe +TabItem +TabSeparated +TabSeparatedRaw +TabSeparatedRawWithNames +TabSeparatedRawWithNamesAndTypes +TabSeparatedWithNames +TabSeparatedWithNamesAndTypes +Tabix +TablesLoaderBackgroundThreads +TablesLoaderBackgroundThreadsActive +TablesLoaderForegroundThreads +TablesLoaderForegroundThreadsActive +TablesToDropQueueSize +TargetSpecific +Telegraf +TemplateIgnoreSpaces +TemporaryFilesForAggregation +TemporaryFilesForJoin +TemporaryFilesForSort +TemporaryFilesUnknown +Testflows +Tgz +Theil's +ThreadMonotonic +ThreadPoolFSReaderThreads +ThreadPoolFSReaderThreadsActive +ThreadPoolRemoteFSReaderThreads +ThreadPoolRemoteFSReaderThreadsActive +ThreadsActive +ThreadsInOvercommitTracker +TimeSeries +TimescaleDB's +Timeunit +TinyLog +Tkachenko +ToASCII +ToCenterChild +ToChildren +ToGeo +ToGeoBoundary +ToIPv +ToParent +ToSnowflake +ToSnowflakeID +ToString +ToUnicode +Toolset +TopK +TotalBytesOfMergeTreeTables +TotalPartsOfMergeTreeTables +TotalPrimaryKeyBytesInMemory +TotalPrimaryKeyBytesInMemoryAllocated +TotalRowsOfMergeTreeTables +TotalTemporaryFiles +Tradeoff +Transactional +Tsai +Tukey +TwoColumnList +UBSan +UDFs +UInt +UIntN +ULID +ULIDStringToDateTime +UMTS +UNDROP +UPDATEs +URIs +URL +URL's +URLDecode +URLEncode +URLHash +URLHierarchy +URLPathHierarchy +USearch +UTCTimestamp +UUIDNumToString +UUIDStringToNum +UUIDToNum +UUIDs +UUIDv +UUid +Uber +Uint +UncompressedCacheBytes +UncompressedCacheCells +UnidirectionalEdgeIsValid +UniqThetaSketch +Updatable +Uppercased +Uptime +Uptrace +UserID +Util +VARCHAR +VIEWs +Vadim +Valgrind +Vectorized +VersionBadge +VersionInteger +VersionedCollapsingMergeTree +VideoContainer +ViewAllLink +VirtualBox +Vose +WALs +WSFG +Welch's +Werror +Wether +WikiStat +WindowView +Winkler +WithCounter +WithFastCounter +WithNames +WithNamesAndTypes +WordNet +WriteBuffer +WriteBuffers +XCode +XHTML +XORs +Xeon +YAML +YAMLRegExpTree +YYYY +YYYYMMDD +YYYYMMDDToDate +YYYYMMDDhhmmssToDateTime +Yandex +Yasm +ZCurve +ZSTDQAT +Zabbix +Zipkin +ZooKeeper +ZooKeeper's +ZooKeeperRequest +ZooKeeperSession +ZooKeeperWatch +ZooKeepers +aarch +accurateCast +accurateCastOrDefault +accurateCastOrNull +acos +acosh +activecube +activerecord +addDate +addDays +addHours +addInterval +addMicroseconds +addMilliseconds +addMinutes +addMonths +addNanoseconds +addQuarters +addSeconds +addTupleOfIntervals +addWeeks +addYears +addr +addressToLine +addressToLineWithInlines +addressToSymbol +adviced +agg +aggThrow +aggregatefunction +aggregatingmergetree +aggregatio +aggretate +aggthrow +aiochclient +allocator +alphaTokens +amplab +analysisOfVariance +analytics +anonymize +anonymized +ansi +anyHeavy +anyIf +anyLast +anyheavy +anylast +appendTrailingCharIfAbsent +approximative +approxtopk +approxtopsum +argMax +argMin +argmax +argmin +arrayAUC +arrayAll +arrayAvg +arrayCompact +arrayConcat +arrayCount +arrayCumSum +arrayCumSumNonNegative +arrayDifference +arrayDistinct +arrayDotProduct +arrayElement +arrayElementOrNull +arrayEnumerate +arrayEnumerateDense +arrayEnumerateDenseRanked +arrayEnumerateUniq +arrayEnumerateUniqRanked +arrayExists +arrayFill +arrayFilter +arrayFirst +arrayFirstIndex +arrayFirstOrNull +arrayFlatten +arrayFold +arrayIntersect +arrayJaccardIndex +arrayJoin +arrayLast +arrayLastIndex +arrayLastOrNull +arrayMap +arrayMax +arrayMin +arrayPartialReverseSort +arrayPartialShuffle +arrayPartialSort +arrayPopBack +arrayPopFront +arrayProduct +arrayPushBack +arrayPushFront +arrayRandomSample +arrayReduce +arrayReduceInRanges +arrayResize +arrayReverse +arrayReverseFill +arrayReverseSort +arrayReverseSplit +arrayRotateLeft +arrayRotateRight +arrayShiftLeft +arrayShiftRight +arrayShingles +arrayShuffle +arraySlice +arraySort +arraySplit +arrayStringConcat +arraySum +arrayUnion +arrayUniq +arrayWithConstant +arrayZip +arrayZipUnaligned +ascii +asin +asinh +assumeNotNull +async +asynch +atan +atanh +atomicity +auth +authenticator +authenticators +autocompletion +autodetect +autodetected +autogen +autogenerate +autogenerated +autogeneration +autostart +avgWeighted +avgweighted +avro +avx +aws +azureBlobStorage +azureBlobStorageCluster +backend +backoff +backticks +backupview +balancer +basename +bcrypt +benchmarking +bfloat +bigrams +binlog +bitAnd +bitCount +bitHammingDistance +bitNot +bitOr +bitPositionsToArray +bitRotateLeft +bitRotateRight +bitShiftLeft +bitShiftRight +bitSlice +bitTest +bitTestAll +bitTestAny +bitXor +bitmapAnd +bitmapAndCardinality +bitmapAndnot +bitmapAndnotCardinality +bitmapBuild +bitmapCardinality +bitmapContains +bitmapHasAll +bitmapHasAny +bitmapMax +bitmapMin +bitmapOr +bitmapOrCardinality +bitmapSubsetInRange +bitmapSubsetLimit +bitmapToArray +bitmapTransform +bitmapXor +bitmapXorCardinality +bitmask +bitmaskToArray +bitmaskToList +bitov +blake +blockNumber +blockSerializedSize +blockSize +blockinfo +blockreader +blocksize +bool +boolean +bools +boringssl +boundingRatio +bozerkins +broadcasted +brotli +bson +bsoneachrow +buffersize +bugfix +buildId +buildable +builtins +byteHammingDistance +byteSize +byteSlice +byteSwap +bytebase +bytesToCutForIPv +cLoki +caConfig +cacheSessions +cachesize +camelCase +capn +capnproto +cardinalities +cardinality +cartesian +cassandra +casted +catboost +catboostEvaluate +categoricalInformationValue +categoricalinformationvalue +cathetus +cbindgen +cbrt +ccache +cctz +ceil +centroid +certificateFile +cetera +cfg +cgroup +cgroups +chadmin +changeDay +changeHour +changeMinute +changeMonth +changeSecond +changeYear +changelog +changelogs +charset +charsets +chconn +cheatsheet +checkouting +checksummed +checksumming +checksums +childern +chproxy +chunksize +cickhouse +cipherList +ciphertext +cityHash +cityhash +ckibana +ckman +clangd +cli +clickcache +clickcat +clickhouse +clickhousedb +clickhousex +clickmate +clickstream +clickvisual +clockhour +clusterAllReplicas +cmake +codebase +codec +codecs +codepoint +codepoints +collapsingmergetree +combinator +combinators +comparising +composable +compressability +concat +concatAssumeInjective +concatWithSeparator +concatWithSeparatorAssumeInjective +cond +conf +config +configs +conformant +congruential +conjuction +conjuctive +connectionId +const +contrib +convertCharset +coroutines +corrMatrix +corrStable +corrmatrix +corrstable +cosineDistance +countDigits +countEqual +countMatches +countMatchesCaseInsensitive +countSubstrings +countSubstringsCaseInsensitive +countSubstringsCaseInsensitiveUTF +covarPop +covarPopMatrix +covarPopStable +covarSamp +covarSampMatrix +covarSampStable +covarStable +covariates +covarpop +covarpopmatrix +covarpopstable +covarsamp +covarsampmatrix +covarsampstable +covid +cpp +cppkafka +cpu +cramersV +cramersVBiasCorrected +cramersv +cramersvbiascorrected +criteo +crlf +croaring +cronjob +cryptocurrencies +cryptocurrency +cryptographic +csv +csvwithnames +csvwithnamesandtypes +ctukey +curdate +currentDatabase +currentProfiles +currentRoles +currentUser +customizable +customizations +customseparated +customseparatedwithnames +customseparatedwithnamesandtypes +cutFragment +cutIPv +cutQueryString +cutQueryStringAndFragment +cutToFirstSignificantSubdomain +cutToFirstSignificantSubdomainCustom +cutToFirstSignificantSubdomainCustomRFC +cutToFirstSignificantSubdomainCustomWithWWW +cutToFirstSignificantSubdomainCustomWithWWWRFC +cutToFirstSignificantSubdomainRFC +cutToFirstSignificantSubdomainWithWWW +cutToFirstSignificantSubdomainWithWWWRFC +cutURLParameter +cutWWW +cyrus +damerauLevenshteinDistance +datacenter +datacenters +datafiles +datagrip +datalens +datanode +dataset +datasets +datasource +datatypes +dateName +dateTime +dateTimeToSnowflake +dateTimeToSnowflakeID +datetime +datetimes +dayofyear +dbal +dbeaver +dbgen +dbms +ddl +deallocated +deallocation +deallocations +debian +decodeHTMLComponent +decodeURLComponent +decodeURLFormComponent +decodeXMLComponent +decompressor +decrypt +decrypted +decrypts +deduplicate +deduplicated +deduplicating +deduplication +defaultProfiles +defaultRoles +defaultValueOfArgumentType +defaultValueOfTypeName +delim +deltaLake +deltaSum +deltaSumTimestamp +deltalake +deltasum +deltasumtimestamp +demangle +denormalize +denormalized +denormalizing +denormals +dequeued +dequeues +deserialization +deserialized +deserializing +dest +destructor +destructors +detectCharset +detectLanguage +detectLanguageMixed +detectLanguageUnknown +detectProgrammingLanguage +detectTonality +determinator +deterministically +dictGet +dictGetAll +dictGetChildren +dictGetDescendant +dictGetHierarchy +dictGetOrDefault +dictGetOrNull +dictGetUUID +dictHas +dictIsIn +disableProtocols +disjunction +disjunctions +displayName +displaySecretsInShowAndSelect +distinctDynamicTypes +distinctJSONPaths +distinctJSONPathsAndTypes +distinctdynamictypes +distinctjsonpaths +distro +divideDecimal +dmesg +doesnt +domainRFC +domainWithoutWWW +domainWithoutWWWRFC +dont +dotProduct +downsampling +dplyr +dragonbox +dropoff +dumpColumnStructure +durations +ecto +editDistance +editDistanceUTF +embeddings +emptyArray +emptyArrayDate +emptyArrayDateTime +emptyArrayFloat +emptyArrayInt +emptyArrayString +emptyArrayToSingle +emptyArrayUInt +enabledProfiles +enabledRoles +encodeURLComponent +encodeURLFormComponent +encodeXMLComponent +encodings +encryptions +endian +endianness +endsWith +endsWithUTF +endswith +enqueued +enum +enum's +enums +erfc +errorCodeToName +etag +evalMLMethod +exFAT +expiryMsec +exponentialMovingAverage +exponentialTimeDecayedAvg +exponentialTimeDecayedCount +exponentialTimeDecayedMax +exponentialTimeDecayedSum +exponentialmovingaverage +expr +exprN +extendedVerification +extractAll +extractAllGroups +extractAllGroupsHorizontal +extractAllGroupsVertical +extractKeyValuePairs +extractKeyValuePairsWithEscaping +extractTextFromHTML +extractURLParameter +extractURLParameterNames +extractURLParameters +failover +farmFingerprint +farmHash +fastops +fcoverage +fibonacci +fifo +fileCluster +filelog +filesystem +filesystemAvailable +filesystemCapacity +filesystemFree +filesystemUnreserved +filesystems +finalizeAggregation +fips +firstLine +firstSignficantSubdomain +firstSignificantSubdomain +firstSignificantSubdomainCustom +firstSignificantSubdomainCustomRFC +firstSignificantSubdomainRFC +fixedstring +flameGraph +flamegraph +flatbuffers +flattenTuple +flink +fluentd +fmtlib +formatDateTime +formatDateTimeInJoda +formatDateTimeInJodaSyntax +formatQuery +formatQuerySingleLine +formatReadableDecimalSize +formatReadableQuantity +formatReadableSize +formatReadableTimeDelta +formatRow +formatRowNoNewline +formated +formatschema +formatter +formatters +fqdn +frac +freezed +fromDaysSinceYearZero +fromModifiedJulianDay +fromModifiedJulianDayOrNull +fromUTCTimestamp +fromUnixTimestamp +fromUnixTimestampInJodaSyntax +fsync +func +fuzzBits +fuzzJSON +fuzzQuery +fuzzer +fuzzers +gRPC +gaugehistogram +gccMurmurHash +gcem +generateRandom +generateRandomStructure +generateSeries +generateSnowflakeID +generateULID +generateUUIDv +geoDistance +geoToH +geoToS +geobase +geobases +geocode +geohash +geohashDecode +geohashEncode +geohashesInBox +geoip +geospatial +getClientHTTPHeader +getMacro +getOSKernelVersion +getServerPort +getSetting +getSizeOfEnumType +getSubcolumn +getTypeSerializationStreams +getblockinfo +getevents +ghcnd +github +glibc +globalIn +globalNotIn +globbing +glushkovds +golang +googletest +grafana +graphitemergetree +graphouse +graphql +greatCircleAngle +greatCircleDistance +greaterOrEquals +greaterorequals +greenspace +groupArray +groupArrayInsertAt +groupArrayIntersect +groupArrayLast +groupArrayMovingAvg +groupArrayMovingSum +groupArraySample +groupArraySorted +groupBitAnd +groupBitOr +groupBitXor +groupBitmap +groupBitmapAnd +groupBitmapOr +groupBitmapXor +groupConcat +groupUniqArray +grouparray +grouparrayinsertat +grouparrayintersect +grouparraylast +grouparraymovingavg +grouparraymovingsum +grouparraysample +grouparraysorted +groupbitand +groupbitmap +groupbitmapand +groupbitmapor +groupbitmapxor +groupbitor +groupbitxor +groupconcat +groupuniqarray +grpc +grpcio +gtest +gtid +gzip +gzipped +hadoop +halfMD +halfday +hardlink +hardlinked +hardlinks +hasAll +hasAny +hasColumnInTable +hasSubsequence +hasSubsequenceCaseInsensitive +hasSubsequenceCaseInsensitiveUTF +hasSubsequenceUTF +hasSubstr +hasThreadFuzzer +hasToken +hasTokenCaseInsensitive +hasTokenCaseInsensitiveOrNull +hasTokenOrNull +hasall +hasany +hashtables +haversine +hdbc +hdfs +hdfsCluster +heredoc +heredocs +hilbertDecode +hilbertEncode +hiveHash +holistics +homebrew +hopEnd +hopStart +horgh +hostName +hostname +hostnames +houseops +hsts +html +http +https +hudi +hyperscan +hypot +hyvor +iTerm +icosahedron +icudata +idempotency +idnaDecode +idnaEncode +ifNotFinite +ifNull +iframe +ilike +incrementing +indexHint +indexOf +infi +inflight +infty +initcap +initcapUTF +initialQueryID +initializeAggregation +injective +innogames +inodes +instantiation +instantiations +intDiv +intDivOrZero +intExp +intHash +integrational +integrations +interserver +intervalLengthSum +invalidCertificateHandler +invariants +invertedindexes +isConstant +isDecimalOverflow +isFinite +isIPAddressInRange +isIPv +isInfinite +isNaN +isNotDistinctFrom +isNotNull +isNull +isNullable +isValidJSON +isValidUTF +isZeroOrNull +iteratively +jaccard +jaccardIndex +jaroSimilarity +jaroWinklerSimilarity +javaHash +javaHashUTF +jbod +jdbc +jemalloc +jeprof +joinGet +joinGetOrNull +json +jsonMergePatch +jsonasobject +jsonasstring +jsoncolumns +jsoncolumnsmonoblock +jsoncompact +jsoncompactcolumns +jsoncompacteachrow +jsoncompacteachrowwithnames +jsoncompacteachrowwithnamesandtypes +jsoncompactstrings +jsoncompactstringseachrow +jsoncompactstringseachrowwithnames +jsoncompactstringseachrowwithnamesandtypes +jsoneachrow +jsoneachrowwithprogress +jsonobjecteachrow +jsonstrings +jsonstringseachrow +jsonstringseachrowwithprogress +jumpConsistentHash +kRing +kafka +kafkaMurmurHash +kafkacat +keepermap +kerberized +kerberos +kernal +keyspace +keytab +kittenhouse +kolmogorovSmirnovTest +kolmogorovsmirnovtest +kolya +konsole +kostik +kostikConsistentHash +kurtPop +kurtSamp +kurtosis +kurtpop +kurtsamp +lagInFrame +laion +lang +laravel +largestTriangleThreeBuckets +latencies +ldap +leadInFrame +leftPad +leftPadUTF +leftUTF +lemmatization +lemmatize +lemmatized +lengthUTF +lessOrEquals +lessorequals +levenshtein +levenshteinDistance +levenshteinDistanceUTF +lexicographically +lgamma +libFuzzer +libc +libcatboost +libcpuid +libcxx +libcxxabi +libdivide +libfarmhash +libfuzzer +libgsasl +libhdfs +libmetrohash +libpq +libpqxx +librdkafka +libs +libunwind +libuv +libvirt +linearizability +linearizable +linearized +lineasstring +linefeeds +lineitem +lineorder +linestring +linux +llvm +loadDefaultCAFile +localhost +localread +loess +logTrace +logagent +loghouse +london +lookups +loongarch +lowCardinalityIndices +lowCardinalityKeys +lowcardinality +lowerUTF +lowercased +lttb +lzma +macOS +mailrugo +mailto +makeDate +makeDateTime +mannWhitneyUTest +mannwhitneyutest +mapAdd +mapAll +mapApply +mapConcat +mapContains +mapContainsKeyLike +mapExists +mapExtractKeyLike +mapFilter +mapFromArrays +mapKeys +mapPartialReverseSort +mapPartialSort +mapPopulateSeries +mapReverseSort +mapSort +mapSubtract +mapUpdate +mapValues +mappedfile +mariadb +matcher +materializedview +maxIntersections +maxIntersectionsPosition +maxMap +maxintersections +maxintersectionsposition +maxmap +maxmind +mdadm +meanZTest +meanztest +mebibytes +memtable +memtables +mergeTreeIndex +mergeable +mergetree +messageID +metacharacters +metasymbols +metrica +metroHash +mfedotov +mflix +minMap +minMappedArrays +minSampleSizeContinuous +minSampleSizeConversion +mindsdb +minimalistic +mininum +miniselect +minmap +minmax +mins +misconfiguration +mispredictions +mlock +mlockall +mmap +mmapped +modularization +moduli +moduloOrZero +mongoc +mongocxx +mongodb +monotonicity +monthName +mortonDecode +mortonEncode +moscow +msgpack +msgpk +multiFuzzyMatchAllIndices +multiFuzzyMatchAny +multiFuzzyMatchAnyIndex +multiIf +multiMatchAllIndices +multiMatchAny +multiMatchAnyIndex +multiSearchAllPositions +multiSearchAllPositionsCaseInsensitive +multiSearchAllPositionsCaseInsensitiveUTF +multiSearchAllPositionsUTF +multiSearchAny +multiSearchAnyCaseInsensitive +multiSearchAnyCaseInsensitiveUTF +multiSearchAnyUTF +multiSearchFirstIndex +multiSearchFirstIndexCaseInsensitive +multiSearchFirstIndexCaseInsensitiveUTF +multiSearchFirstIndexUTF +multiSearchFirstPosition +multiSearchFirstPositionCaseInsensitive +multiSearchFirstPositionCaseInsensitiveUTF +multiSearchFirstPositionUTF +multibyte +multidirectory +multiline +multilinestring +multiplyDecimal +multipolygon +multisearchany +multisets +multithread +multiword +munmap +murmurHash +murmurhash +musqldump +mutex +mydb +myfilter +mysql +mysqldump +mysqljs +mytable +namedatabases +namenetworks +namenode +namepassword +nameprofile +namequota +namespace +namespaces +natively +nats +ness +nestjs +netloc +newjson +ngram +ngramDistance +ngramDistanceCaseInsensitive +ngramDistanceCaseInsensitiveUTF +ngramDistanceUTF +ngramMinHash +ngramMinHashArg +ngramMinHashArgCaseInsensitive +ngramMinHashArgCaseInsensitiveUTF +ngramMinHashArgUTF +ngramMinHashCaseInsensitive +ngramMinHashCaseInsensitiveUTF +ngramMinHashUTF +ngramSearch +ngramSearchCaseInsensitive +ngramSearchCaseInsensitiveUTF +ngramSearchUTF +ngramSimHash +ngramSimHashCaseInsensitive +ngramSimHashCaseInsensitiveUTF +ngramSimHashUTF +ngrambf +ngrams +noaa +nonNegativeDerivative +noop +normalizeQuery +normalizeQueryKeepNames +normalizeUTF +normalizedQueryHash +normalizedQueryHashKeepNames +notEmpty +notEquals +notILike +notIn +notLike +notempty +notequals +notlike +notretry +nowInBlock +ntile +nullIf +nullability +nullable +nullables +num +numerics +nypd +obfuscator +observability +odbc +ok +omclickhouse +onstraints +ontime +onwards +openSSL +openSUSE +openldap +opensky +openssl +opentelemetry +outfile +overcommit +overcommitted +overfitting +overlayUTF +overparallelization +packetpool +packetsize +pageviews +parallelization +parallelize +parallelized +params +parseDateTime +parseDateTimeBestEffort +parseDateTimeBestEffortOrNull +parseDateTimeBestEffortOrZero +parseDateTimeBestEffortUS +parseDateTimeBestEffortUSOrNull +parseDateTimeBestEffortUSOrZero +parseDateTimeInJodaSyntax +parseDateTimeInJodaSyntaxOrNull +parseDateTimeInJodaSyntaxOrZero +parseDateTimeOrNull +parseDateTimeOrZero +parseReadableSize +parseReadableSizeOrNull +parseReadableSizeOrZero +parseTimeDelta +parseable +parsers +partitionID +partitionId +pathFull +pclmulqdq +pcre +performant +perl +persistency +phpclickhouse +pipelining +plaintext +plantuml +poco +pointInEllipses +pointInPolygon +polygonAreaCartesian +polygonAreaSpherical +polygonConvexHullCartesian +polygonPerimeterCartesian +polygonPerimeterSpherical +polygonsDistanceCartesian +polygonsDistanceSpherical +polygonsEqualsCartesian +polygonsIntersectionCartesian +polygonsIntersectionSpherical +polygonsSymDifferenceCartesian +polygonsSymDifferenceSpherical +polygonsUnionCartesian +polygonsUnionSpherical +polygonsWithinCartesian +polygonsWithinSpherical +popcnt +portRFC +porthttps +positionCaseInsensitive +positionCaseInsensitiveUTF +positionUTF +positiveModulo +postfix +postfixes +postgresql +pre +pread +preallocate +prebuild +prebuilt +preemptable +preferServerCiphers +prefetch +prefetchsize +preloaded +prem +prepend +prepended +prepends +preprocess +preprocessed +preprocessing +preprocessor +presentational +prestable +prettycompact +prettycompactmonoblock +prettycompactnoescapes +prettycompactnoescapesmonoblock +prettyjsoneachrow +prettymonoblock +prettynoescapes +prettynoescapesmonoblock +prettyspace +prettyspacemonoblock +prettyspacenoescapes +prettyspacenoescapesmonoblock +prewhere +printf +privateKeyFile +privateKeyPassphraseHandler +prlimit +procfs +profiler +proleptic +prometheus +proportionsZTest +proto +protobuf +protobuflist +protobufsingle +protocol +proxied +pseudorandom +pseudorandomize +psql +ptrs +punycodeDecode +punycodeEncode +pushdown +pwrite +py +qryn +quantile +quantileBFloat +quantileDD +quantileDeterministic +quantileExact +quantileExactExclusive +quantileExactHigh +quantileExactInclusive +quantileExactLow +quantileExactWeighted +quantileGK +quantileInterpolatedWeighted +quantileTDigest +quantileTDigestWeighted +quantileTiming +quantileTimingWeighted +quantilebfloat +quantileddsketch +quantiledeterministic +quantileexact +quantileexactweighted +quantiles +quantilesExactExclusive +quantilesExactInclusive +quantilesGK +quantilesTimingWeighted +quantiletdigest +quantiletdigestweighted +quantiletiming +quantiletimingweighted +quartile +queryID +queryString +queryStringAndFragment +rabbitmq +raduis +randBernoulli +randBinomial +randCanonical +randChiSquared +randConstant +randExponential +randFisherF +randLogNormal +randNegativeBinomial +randNormal +randPoisson +randStudentT +randUniform +randomFixedString +randomPrintableASCII +randomString +randomStringUTF +rankCorr +rapidjson +rawblob +readWKTLineString +readWKTMultiLineString +readWKTMultiPolygon +readWKTPoint +readWKTPolygon +readWKTRing +readahead +readline +readme +readonly +rebalance +rebalanced +recency +recompress +recompressed +recompressing +recompression +reconnection +recurse +redash +reddit +redis +redisstreams +refcounter +refreshable +regexpExtract +regexpQuoteMeta +regionHierarchy +regionIn +regionToArea +regionToCity +regionToContinent +regionToCountry +regionToDistrict +regionToName +regionToPopulation +regionToTopContinent +reinitialization +reinitializing +reinterpretAs +reinterpretAsDate +reinterpretAsDateTime +reinterpretAsFixedString +reinterpretAsFloat +reinterpretAsInt +reinterpretAsString +reinterpretAsUInt +reinterpretAsUUID +remoteSecure +repivot +replaceAll +replaceOne +replaceRegexpAll +replaceRegexpOne +replacingmergetree +replicatable +replicatedmergetree +replxx +repo +representable +requestor +requireTLSv +resharding +reshards +resolvers +resultset +retentions +rethrow +retransmit +retriable +reverseUTF +rewritable +rightPad +rightPadUTF +rightUTF +risc +riscv +ro +roadmap +rocksdb +rollup +roundAge +roundBankers +roundDown +roundDuration +roundToExp +routineley +rowNumberInAllBlocks +rowNumberInBlock +rowbinary +rowbinarywithdefaults +rowbinarywithnames +rowbinarywithnamesandtypes +rsync +rsyslog +runnable +runningAccumulate +runningConcurrency +runningDifference +runningDifferenceStartingWithFirstValue +runtime +russian +rustc +rustup +rw +sasl +satisfiable +scala +sccache +schemas +seekable +seektable +sequenceCount +sequenceMatch +sequenceNextNode +seriesDecomposeSTL +seriesOutliersDetectTukey +seriesPeriodDetectFFT +serverTimeZone +serverTimezone +serverUUID +sessionCacheSize +sessionIdContext +sessionTimeout +seva +shardCount +shardNum +sharded +sharding +shortcircuit +shortkeys +shoutout +showCertificate +sigmoid +simdjson +simpleJSON +simpleJSONExtractBool +simpleJSONExtractFloat +simpleJSONExtractInt +simpleJSONExtractRaw +simpleJSONExtractString +simpleJSONExtractUInt +simpleJSONHas +simpleLinearRegression +simpleaggregatefunction +simplelinearregression +simpod +singleValueOrNull +singlepart +singlevalueornull +sinh +sipHash +siphash +skewPop +skewSamp +skewness +skewpop +skewsamp +skippingerrors +sleepEachRow +snowflakeIDToDateTime +snowflakeToDateTime +socketcache +soundex +sparkBar +sparkbar +sparsehash +speedscope +splitBy +splitByChar +splitByNonAlpha +splitByRegexp +splitByString +splitByWhitespace +splitby +sqid +sqidDecode +sqidEncode +sql +sqlalchemy +sqlinsert +sqlite +sqrt +src +srcReplicas +sshkey +stackoverflow +stacktrace +stacktraces +startsWith +startsWithUTF +startswith +statbox +stateful +stateset +stddev +stddevPop +stddevPopStable +stddevSamp +stddevSampStable +stddevpop +stddevpopstable +stddevsamp +stddevsampstable +stderr +stdin +stdout +stochasticLinearRegression +stochasticLogisticRegression +stochastically +stochasticlinearregression +stochasticlogisticregression +storages +storig +stringJaccardIndex +stringJaccardIndexUTF +stringToH +stripelog +strtod +strtoll +strtoull +struct +structs +structureToCapnProtoSchema +structureToProtobufSchema +studentTTest +studentttest +subBitmap +subDate +subarray +subarrays +subcolumn +subcolumns +subdirectories +subdirectory +subexpression +subexpressions +subfolder +subfolders +subinterval +subintervals +subkey +submatch +submodule +submodules +subnet +subnetwork +subpattern +subpatterns +subqueries +subquery +subranges +subreddits +subseconds +subsequence +substreams +substring +substringIndex +substringIndexUTF +substringUTF +substrings +subtitiles +subtractDays +subtractHours +subtractInterval +subtractMicroseconds +subtractMilliseconds +subtractMinutes +subtractMonths +subtractNanoseconds +subtractQuarters +subtractSeconds +subtractTupleOfIntervals +subtractWeeks +subtractYears +subtree +subtrees +subtype +sudo +sumCount +sumKahan +sumMap +sumMapFiltered +sumMapFilteredWithOverflow +sumMapWithOverflow +sumWithOverflow +sumcount +sumkahan +summap +summapwithoverflow +summingmergetree +sumwithoverflow +superaggregates +supertype +supremum +symlink +symlinks +syntaxes +syscall +syscalls +sysctl +syslog +syslogd +systemd +tabix +tablum +tabseparated +tabseparatedraw +tabseparatedrawwithnames +tabseparatedrawwithnamesandtypes +tabseparatedwithnames +tabseparatedwithnamesandtypes +tanh +tcp +tcpPort +tcpnodelay +templateignorespaces +tgamma +tgz +th +theilsU +theilsu +themself +threadpool +throwIf +timeDiff +timeSeriesData +timeSeriesMetrics +timeSeriesTags +timeSlot +timeSlots +timeZone +timeZoneOf +timeZoneOffset +timezones +tinylog +tmp +toBool +toColumnTypeName +toDate +toDateOrDefault +toDateOrNull +toDateOrZero +toDateTime +toDateTimeOrDefault +toDateTimeOrNull +toDateTimeOrZero +toDayOfMonth +toDayOfWeek +toDayOfYear +toDaysSinceYearZero +toDecimal +toDecimalString +toFixedString +toFloat +toHour +toIPv +toISOWeek +toISOYear +toInt +toInterval +toIntervalDay +toIntervalHour +toIntervalMicrosecond +toIntervalMillisecond +toIntervalMinute +toIntervalMonth +toIntervalNanosecond +toIntervalQuarter +toIntervalSecond +toIntervalWeek +toIntervalYear +toJSONString +toLastDayOfMonth +toLastDayOfWeek +toLowCardinality +toMillisecond +toMinute +toModifiedJulianDay +toModifiedJulianDayOrNull +toMonday +toMonth +toNullable +toQuarter +toRelativeDayNum +toRelativeHourNum +toRelativeMinuteNum +toRelativeMonthNum +toRelativeQuarterNum +toRelativeSecondNum +toRelativeWeekNum +toRelativeYearNum +toSecond +toStartOfDay +toStartOfFifteenMinutes +toStartOfFiveMinutes +toStartOfHour +toStartOfISOYear +toStartOfInterval +toStartOfMicrosecond +toStartOfMillisecond +toStartOfMinute +toStartOfMonth +toStartOfNanosecond +toStartOfQuarter +toStartOfSecond +toStartOfTenMinutes +toStartOfWeek +toStartOfYear +toString +toStringCutToZero +toTime +toTimeZone +toType +toTypeName +toUInt +toUTCTimestamp +toUUID +toUUIDOrDefault +toUUIDOrNull +toUUIDOrZero +toUnixTimestamp +toValidUTF +toWeek +toYYYYMM +toYYYYMMDD +toYYYYMMDDhhmmss +toYear +toYearWeek +tokenbf +tokenization +tokenized +tokenizer +toml +toolchain +toolset +topK +topKWeighted +topLevelDomain +topLevelDomainRFC +topk +topkweighted +tpcds +tpch +transactionID +transactionLatestSnapshot +transactionOldestSnapshot +transactional +transactionally +translateUTF +translocality +trie +trimBoth +trimLeft +trimRight +trunc +tryBase +tryDecrypt +tryIdnaEncode +tryPunycodeDecode +tskv +tsv +tui +tukey +tumbleEnd +tumbleStart +tupleConcat +tupleDivide +tupleDivideByNumber +tupleElement +tupleHammingDistance +tupleIntDiv +tupleIntDivByNumber +tupleIntDivOrZero +tupleIntDivOrZeroByNumber +tupleMinus +tupleModulo +tupleModuloByNumber +tupleMultiply +tupleMultiplyByNumber +tupleNames +tupleNegate +tuplePlus +tupleToNameValuePairs +turbostat +txt +typename +ubuntu +uint +ulid +unary +unbin +uncomment +undrop +unencoded +unencrypted +unescaped +unescaping +unhex +unicode +unidimensional +unigrams +unintuitive +uniq +uniqCombined +uniqExact +uniqHLL +uniqTheta +uniqThetaIntersect +uniqThetaNot +uniqThetaSketch +uniqThetaUnion +uniqUpTo +uniqcombined +uniqexact +uniqhll +uniqtheta +uniqthetasketch +unix +unixODBC +unixodbc +unoptimized +unparsed +unpooled +unrealiable +unreplicated +unresolvable +unrounded +unshuffled +untracked +untrusted +untuple +uploader +uploaders +upperUTF +uptime +uptrace +uring +url +urlCluster +urlencoded +urls +usearch +userspace +userver +utils +uuid +uuidv +vCPU +varPop +varPopStable +varSamp +varSampStable +variadic +variantElement +variantType +varint +varpop +varpopstable +varsamp +varsampstable +vectorized +vectorscan +vendoring +verificationDepth +verificationMode +versionedcollapsingmergetree +vhost +virtualized +visibleWidth +visitParam +visitParamExtractBool +visitParamExtractFloat +visitParamExtractInt +visitParamExtractRaw +visitParamExtractString +visitParamExtractUInt +visitParamHas +vruntime +wchc +wchs +webpage +webserver +weekyear +welchTTest +welchttest +wget +which's +whitespace +whitespaces +wikistat +windowFunnel +wordShingleMinHash +wordShingleMinHashArg +wordShingleMinHashArgCaseInsensitive +wordShingleMinHashArgCaseInsensitiveUTF +wordShingleMinHashArgUTF +wordShingleMinHashCaseInsensitive +wordShingleMinHashCaseInsensitiveUTF +wordShingleMinHashUTF +wordShingleSimHash +wordShingleSimHashCaseInsensitive +wordShingleSimHashCaseInsensitiveUTF +wordShingleSimHashUTF +wordshingleMinHash +writability +wrt +wyHash +xcode +xeus +xkcd +xlarge +xml +xxHash +xz +yaml +yandex +youtube +zLib +zLinux +zabbix +zkcopy +zlib +znode +znodes +zookeeperSessionUptime +zstd diff --git a/ci_v2/jobs/scripts/check_style/check_aspell.sh b/ci_v2/jobs/scripts/check_style/check_aspell.sh new file mode 100755 index 00000000000..61726aab0f0 --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/check_aspell.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# force-enable double star globbing +shopt -s globstar + +# Perform spell checking on the docs + +if [[ ${1:-} == "--help" ]] || [[ ${1:-} == "-h" ]]; then + echo "Usage $0 [--help|-h] [-i [filename]]" + echo " --help|-h: print this help" + echo " -i: interactive mode. If filename is specified, check only this file, otherwise check all files" + exit 0 +fi + +ROOT_PATH="." + +CHECK_LANG=en + +ASPELL_IGNORE_PATH="${ROOT_PATH}/utils/check-style/aspell-ignore/${CHECK_LANG}" + +if [[ ${1:-} == "-i" ]]; then + if [[ ! -z ${2:-} ]]; then + FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/${2} + else + FILES_TO_CHECK=${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md + fi + for fname in ${FILES_TO_CHECK}; do + echo "Checking $fname" + aspell --personal=aspell-dict.txt --add-sgml-skip=code --encoding=utf-8 --mode=markdown -W 3 --lang=${CHECK_LANG} --home-dir=${ASPELL_IGNORE_PATH} -c "$fname" + done + exit +fi + +STATUS=0 +for fname in ${ROOT_PATH}/docs/${CHECK_LANG}/**/*.md; do + errors=$(cat "$fname" \ + | aspell list \ + -W 3 \ + --personal=aspell-dict.txt \ + --add-sgml-skip=code \ + --encoding=utf-8 \ + --mode=markdown \ + --lang=${CHECK_LANG} \ + --home-dir=${ASPELL_IGNORE_PATH} \ + | sort | uniq) + if [ ! -z "$errors" ]; then + STATUS=1 + echo "====== $fname ======" + echo "$errors" + fi +done + +if (( STATUS != 0 )); then + echo "====== Errors found ======" + echo "To exclude some words add them to the dictionary file \"${ASPELL_IGNORE_PATH}/aspell-dict.txt\"" + echo "You can also run ${0} -i to see the errors interactively and fix them or add to the dictionary file" +fi + +exit ${STATUS} diff --git a/ci_v2/jobs/scripts/check_style/check_cpp.sh b/ci_v2/jobs/scripts/check_style/check_cpp.sh new file mode 100755 index 00000000000..1611fac8c5e --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/check_cpp.sh @@ -0,0 +1,339 @@ +#!/usr/bin/env bash + +# For code formatting we have clang-format. +# +# But it's not sane to apply clang-format for whole code base, +# because it sometimes makes worse for properly formatted files. +# +# It's only reasonable to blindly apply clang-format only in cases +# when the code is likely to be out of style. +# +# For this purpose we have a script that will use very primitive heuristics +# (simple regexps) to check if the code is likely to have basic style violations. +# and then to run formatter only for the specified files. + +LC_ALL="en_US.UTF-8" +ROOT_PATH="." +EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|utils/keeper-bench/example.yaml' + +# From [1]: +# But since array_to_string_internal() in array.c still loops over array +# elements and concatenates them into a string, it's probably not more +# efficient than the looping solutions proposed, but it's more readable. +# +# [1]: https://stackoverflow.com/a/15394738/328260 +function in_array() +{ + local IFS="|" + local value=$1 && shift + + [[ "${IFS}${*}${IFS}" =~ "${IFS}${value}${IFS}" ]] +} + +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null | + grep -vP $EXCLUDE_DIRS | + xargs grep $@ -P '((class|struct|namespace|enum|if|for|while|else|throw|switch).*|\)(\s*const)?(\s*override)?\s*)\{$|\s$|^ {1,3}[^\* ]\S|\t|^\s*(if|else if|if constexpr|else if constexpr|for|while|catch|switch)\(|\( [^\s\\]|\S \)' | +# a curly brace not in a new line, but not for the case of C++11 init or agg. initialization | trailing whitespace | number of ws not a multiple of 4, but not in the case of comment continuation | missing whitespace after for/if/while... before opening brace | whitespaces inside braces + grep -v -P '(//|:\s+\*|\$\(\()| \)"' +# single-line comment | continuation of a multiline comment | a typical piece of embedded shell code | something like ending of raw string literal + +# Tabs +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null | + grep -vP $EXCLUDE_DIRS | + xargs grep $@ -F $'\t' + +# // namespace comments are unneeded +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null | + grep -vP $EXCLUDE_DIRS | + xargs grep $@ -P '}\s*//+\s*namespace\s*' + +# Broken symlinks +find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found" + +# Duplicated or incorrect setting declarations +SETTINGS_FILE=$(mktemp) +cat $ROOT_PATH/src/Core/Settings.cpp $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " " substr($1, 3, length($1) - 3) " SettingsDeclaration" }' > ${SETTINGS_FILE} +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep "extern const Settings" -T | awk '{print substr($5, 0, length($5) -1) " " substr($4, 9) " " substr($1, 0, length($1) - 1)}' >> ${SETTINGS_FILE} + +# Duplicate extern declarations for settings +awk '{if (seen[$0]++) print $3 " -> " $1 ;}' ${SETTINGS_FILE} | while read line; +do + echo "Found duplicated setting declaration in: $line" +done + +# Incorrect declarations for settings +for setting in $(awk '{print $1 " " $2}' ${SETTINGS_FILE} | sort | uniq | awk '{ print $1 }' | sort | uniq -d); +do + expected=$(grep "^$setting " ${SETTINGS_FILE} | grep SettingsDeclaration | awk '{ print $2 }') + grep "^$setting " ${SETTINGS_FILE} | grep -v " $expected" | awk '{ print $3 " found setting " $1 " with type " $2 }' | while read line; + do + echo "In $line but it should be $expected" + done +done + +rm ${SETTINGS_FILE} + +# Unused/Undefined/Duplicates ErrorCodes/ProfileEvents/CurrentMetrics +declare -A EXTERN_TYPES +EXTERN_TYPES[ErrorCodes]=int +EXTERN_TYPES[ProfileEvents]=Event +EXTERN_TYPES[CurrentMetrics]=Metric + +EXTERN_TYPES_EXCLUDES=( + ProfileEvents::global_counters + ProfileEvents::Event + ProfileEvents::Count + ProfileEvents::Counters + ProfileEvents::end + ProfileEvents::increment + ProfileEvents::incrementForLogMessage + ProfileEvents::getName + ProfileEvents::Timer + ProfileEvents::Type + ProfileEvents::TypeEnum + ProfileEvents::dumpToMapColumn + ProfileEvents::getProfileEvents + ProfileEvents::ThreadIdToCountersSnapshot + ProfileEvents::LOCAL_NAME + ProfileEvents::keeper_profile_events + ProfileEvents::CountersIncrement + + CurrentMetrics::add + CurrentMetrics::sub + CurrentMetrics::get + CurrentMetrics::set + CurrentMetrics::end + CurrentMetrics::Increment + CurrentMetrics::Metric + CurrentMetrics::values + CurrentMetrics::Value + CurrentMetrics::keeper_metrics + + ErrorCodes::ErrorCode + ErrorCodes::getName + ErrorCodes::increment + ErrorCodes::end + ErrorCodes::values + ErrorCodes::values[i] + ErrorCodes::getErrorCodeByName + ErrorCodes::Value +) +for extern_type in ${!EXTERN_TYPES[@]}; do + type_of_extern=${EXTERN_TYPES[$extern_type]} + allowed_chars='[_A-Za-z]+' + + # Unused + # NOTE: to fix automatically, replace echo with: + # sed -i "/extern const $type_of_extern $val/d" $file + find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | { + # NOTE: the check is pretty dumb and distinguish only by the type_of_extern, + # and this matches with zkutil::CreateMode + grep -v -e 'src/Common/ZooKeeper/Types.h' -e 'src/Coordination/KeeperConstants.cpp' + } | { + grep -vP $EXCLUDE_DIRS | xargs grep -l -P "extern const $type_of_extern $allowed_chars" + } | while read file; do + grep -P "extern const $type_of_extern $allowed_chars;" $file | sed -r -e "s/^.*?extern const $type_of_extern ($allowed_chars);.*?$/\1/" | while read val; do + if ! grep -q "$extern_type::$val" $file; then + # Excludes for SOFTWARE_EVENT/HARDWARE_EVENT/CACHE_EVENT in ThreadProfileEvents.cpp + if [[ ! $extern_type::$val =~ ProfileEvents::Perf.* ]]; then + echo "$extern_type::$val is defined but not used in file $file" + fi + fi + done + done + + # Undefined + # NOTE: to fix automatically, replace echo with: + # ( grep -q -F 'namespace $extern_type' $file && \ + # sed -i -r "0,/(\s*)extern const $type_of_extern [$allowed_chars]+/s//\1extern const $type_of_extern $val;\n&/" $file || \ + # awk '{ print; if (ns == 1) { ns = 2 }; if (ns == 2) { ns = 0; print "namespace $extern_type\n{\n extern const $type_of_extern '$val';\n}" } }; /namespace DB/ { ns = 1; };' < $file > ${file}.tmp && mv ${file}.tmp $file ) + find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | { + grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars" + } | while read file; do + grep -P "$extern_type::$allowed_chars" $file | grep -P -v '^\s*//' | sed -r -e "s/^.*?$extern_type::($allowed_chars).*?$/\1/" | while read val; do + if ! grep -q "extern const $type_of_extern $val" $file; then + if ! in_array "$extern_type::$val" "${EXTERN_TYPES_EXCLUDES[@]}"; then + echo "$extern_type::$val is used in file $file but not defined" + fi + fi + done + done + + # Duplicates + find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | { + grep -vP $EXCLUDE_DIRS | xargs grep -l -P "$extern_type::$allowed_chars" + } | while read file; do + grep -P "extern const $type_of_extern $allowed_chars;" $file | sort | uniq -c | grep -v -P ' +1 ' && echo "Duplicate $extern_type in file $file" + done +done + +# Three or more consecutive empty lines +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + while read file; do awk '/^$/ { ++i; if (i > 2) { print "More than two consecutive empty lines in file '$file'" } } /./ { i = 0 }' $file; done + +# Check that every header file has #pragma once in first line +find $ROOT_PATH/{src,programs,utils} -name '*.h' | + grep -vP $EXCLUDE_DIRS | + while read file; do [[ $(head -n1 $file) != '#pragma once' ]] && echo "File $file must have '#pragma once' in first line"; done + +# Too many exclamation marks +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -F '!!!' | grep -P '.' && echo "Too many exclamation marks (looks dirty, unconfident)." + +# Exclamation mark in a message +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -F '!",' | grep -P '.' && echo "No need for an exclamation mark (looks dirty, unconfident)." + +# Trailing whitespaces +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -n -P ' $' | grep -n -P '.' && echo "^ Trailing whitespaces." + +# Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -P 'std::[io]?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream" + +# Forbid std::cerr/std::cout in src (fine in programs/utils) +std_cerr_cout_excludes=( + /examples/ + /tests/ + _fuzzer + # OK + src/Common/ProgressIndication.cpp + # only under #ifdef DBMS_HASH_MAP_DEBUG_RESIZES, that is used only in tests + src/Common/HashTable/HashTable.h + # SensitiveDataMasker::printStats() + src/Common/SensitiveDataMasker.cpp + # StreamStatistics::print() + src/Compression/LZ4_decompress_faster.cpp + # ContextSharedPart with subsequent std::terminate() + src/Interpreters/Context.cpp + # IProcessor::dump() + src/Processors/IProcessor.cpp + src/Client/ClientApplicationBase.cpp + src/Client/ClientBase.cpp + src/Client/LineReader.cpp + src/Client/QueryFuzzer.cpp + src/Client/Suggest.cpp + src/Client/ClientBase.h + src/Client/LineReader.h + src/Client/ReplxxLineReader.h + src/Bridge/IBridge.cpp + src/Daemon/BaseDaemon.cpp + src/Loggers/Loggers.cpp + src/Common/GWPAsan.cpp + src/Common/ProgressIndication.h +) +sources_with_std_cerr_cout=( $( + find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \ + grep -vP $EXCLUDE_DIRS | \ + grep -F -v $(printf -- "-e %s " "${std_cerr_cout_excludes[@]}") | \ + xargs grep -F --with-filename -e std::cerr -e std::cout | cut -d: -f1 | sort -u +) ) + +# Exclude comments +for src in "${sources_with_std_cerr_cout[@]}"; do + # suppress stderr, since it may contain warning for #pargma once in headers + if gcc -fpreprocessed -dD -E "$src" 2>/dev/null | grep -F -q -e std::cerr -e std::cout; then + echo "$src: uses std::cerr/std::cout" + fi +done + +expect_tests=( $(find $ROOT_PATH/tests/queries -name '*.expect') ) +for test_case in "${expect_tests[@]}"; do + pattern="^exp_internal -f \$CLICKHOUSE_TMP/\$basename.debuglog 0$" + grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" + + if grep -q "^spawn.*CLICKHOUSE_CLIENT_BINARY$" "$test_case"; then + pattern="^spawn.*CLICKHOUSE_CLIENT_BINARY.*--history_file$" + grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" + fi + + # Otherwise expect_after/expect_before will not bail without stdin attached + # (and actually this is a hack anyway, correct way is to use $any_spawn_id) + pattern="-i \$any_spawn_id timeout" + grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" + pattern="-i \$any_spawn_id eof" + grep -q -- "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" +done + +# Forbid non-unique error codes +if [[ "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | wc -l)" != "$(grep -Po "M\([0-9]*," $ROOT_PATH/src/Common/ErrorCodes.cpp | sort | uniq | wc -l)" ]] +then + echo "ErrorCodes.cpp contains non-unique error codes" +fi + +# Check that there is no system-wide libraries/headers in use. +# +# NOTE: it is better to override find_path/find_library in cmake, but right now +# it is not possible, see [1] for the reference. +# +# [1]: git grep --recurse-submodules -e find_library -e find_path contrib +if git grep -e find_path -e find_library -- :**CMakeLists.txt; then + echo "There is find_path/find_library usage. ClickHouse should use everything bundled. Consider adding one more contrib module." +fi + +# Forbid std::filesystem::is_symlink and std::filesystem::read_symlink, because it's easy to use them incorrectly +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -P '::(is|read)_symlink' | grep -v "STYLE_CHECK_ALLOW_STD_FS_SYMLINK" && echo "Use DB::FS::isSymlink and DB::FS::readSymlink instead" + +# Forbid __builtin_unreachable(), because it's hard to debug when it becomes reachable +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -P '__builtin_unreachable' && echo "Use UNREACHABLE() from defines.h instead" + +# Forbid mt19937() and random_device() which are outdated and slow +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -P '(std::mt19937|std::mersenne_twister_engine|std::random_device)' && echo "Use pcg64_fast (from pcg_random.h) and randomSeed (from Common/randomSeed.h) instead" + +# Require checking return value of close(), +# since it can hide fd misuse and break other places. +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -e ' close(.*fd' -e ' ::close(' | grep -v = && echo "Return value of close() should be checked" + +# A small typo can lead to debug code in release builds, see https://github.com/ClickHouse/ClickHouse/pull/47647 +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -l -F '#ifdef NDEBUG' | xargs -I@FILE awk '/#ifdef NDEBUG/ { inside = 1; dirty = 1 } /#endif/ { if (inside && dirty) { print "File @FILE has suspicious #ifdef NDEBUG, possibly confused with #ifndef NDEBUG" }; inside = 0 } /#else/ { dirty = 0 }' @FILE + +# If a user is doing dynamic or typeid cast with a pointer, and immediately dereferencing it, it is unsafe. +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep --line-number -P '(dynamic|typeid)_cast<[^>]+\*>\([^\(\)]+\)->' | grep -P '.' && echo "It's suspicious when you are doing a dynamic_cast or typeid_cast with a pointer and immediately dereferencing it. Use references instead of pointers or check a pointer to nullptr." + +# Check for bad punctuation: whitespace before comma. +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number '\w ,' | grep -v 'bad punctuation is ok here' && echo "^ There is bad punctuation: whitespace before comma. You should write it like this: 'Hello, world!'" + +# Check usage of std::regex which is too bloated and slow. +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P --line-number 'std::regex' | grep -P '.' && echo "^ Please use re2 instead of std::regex" + +# Cyrillic characters hiding inside Latin. +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | grep -v StorageSystemContributors.generated.cpp | xargs grep -P --line-number '[a-zA-Z][а-яА-ЯёЁ]|[а-яА-ЯёЁ][a-zA-Z]' && echo "^ Cyrillic characters found in unexpected place." + +# Orphaned header files. +join -v1 <(find $ROOT_PATH/{src,programs,utils} -name '*.h' -printf '%f\n' | sort | uniq) <(find $ROOT_PATH/{src,programs,utils} -name '*.cpp' -or -name '*.c' -or -name '*.h' -or -name '*.S' | xargs grep --no-filename -o -P '[\w-]+\.h' | sort | uniq) | + grep . && echo '^ Found orphan header files.' + +# Don't allow dynamic compiler check with CMake, because we are using hermetic, reproducible, cross-compiled, static (TLDR, good) builds. +ls -1d $ROOT_PATH/contrib/*-cmake | xargs -I@ find @ -name 'CMakeLists.txt' -or -name '*.cmake' | xargs grep --with-filename -i -P 'check_c_compiler_flag|check_cxx_compiler_flag|check_c_source_compiles|check_cxx_source_compiles|check_include_file|check_symbol_exists|cmake_push_check_state|cmake_pop_check_state|find_package|CMAKE_REQUIRED_FLAGS|CheckIncludeFile|CheckCCompilerFlag|CheckCXXCompilerFlag|CheckCSourceCompiles|CheckCXXSourceCompiles|CheckCSymbolExists|CheckCXXSymbolExists' | grep -v Rust && echo "^ It's not allowed to have dynamic compiler checks with CMake." + +# Wrong spelling of abbreviations, e.g. SQL is right, Sql is wrong. XMLHttpRequest is very wrong. +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -P 'Sql|Html|Xml|Cpu|Tcp|Udp|Http|Db|Json|Yaml' | grep -v -P 'RabbitMQ|Azure|Aws|aws|Avro|IO/S3' && + echo "Abbreviations such as SQL, XML, HTTP, should be in all caps. For example, SQL is right, Sql is wrong. XMLHttpRequest is very wrong." + +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | + grep -vP $EXCLUDE_DIRS | + xargs grep -F -i 'ErrorCodes::LOGICAL_ERROR, "Logical error:' && + echo "If an exception has LOGICAL_ERROR code, there is no need to include the text 'Logical error' in the exception message, because then the phrase 'Logical error' will be printed twice." + +# There shouldn't be any code snippets under GPL or LGPL +find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL" + +PATTERN="allow_"; +DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.cpp | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u)); +[ -n "$DIFF" ] && echo "$DIFF" && echo "^^ Detected 'allow_*' settings that might need to be included in src/Databases/enableAllExperimentalSettings.cpp" && echo "Alternatively, consider adding an exception to utils/check-style/experimental_settings_ignore.txt" diff --git a/ci_v2/jobs/scripts/check_style/check_submodules.sh b/ci_v2/jobs/scripts/check_style/check_submodules.sh new file mode 100755 index 00000000000..eeb893b9615 --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/check_submodules.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# The script checks if all submodules defined in $GIT_ROOT/.gitmodules exist in $GIT_ROOT/contrib + +set -e + +GIT_ROOT="." + +cd "$GIT_ROOT" + +# Remove keys for submodule.*.path parameters, the values are separated by \0 +# and check if the directory exists +git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \ + xargs -P100 -0 --no-run-if-empty -I{} bash -c 'if ! test -d '"'{}'"'; then echo Directory for submodule {} is not found; exit 1; fi' 2>&1 + + +# And check that the submodule is fine +git config --file .gitmodules --null --get-regexp path | sed -z 's|.*\n||' | \ + xargs -P100 -0 --no-run-if-empty -I{} git submodule status -q '{}' 2>&1 + + +# All submodules should be from https://github.com/ +git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.url' | \ +while read -r line; do + name=${line#submodule.}; name=${name%.url*} + url=${line#* } + [[ "$url" != 'https://github.com/'* ]] && echo "All submodules should be from https://github.com/, submodule '$name' has '$url'" +done + +# All submodules should be of this form: [submodule "contrib/libxyz"] (for consistency, the submodule name does matter too much) +# - restrict the check to top-level .gitmodules file +git config --file "$ROOT_PATH/.gitmodules" --get-regexp 'submodule\..+\.path' | \ +while read -r line; do + name=${line#submodule.}; name=${name%.path*} + path=${line#* } + [ "$name" != "$path" ] && echo "Submodule name '$name' is not equal to it's path '$path'" +done diff --git a/ci_v2/jobs/scripts/check_style/check_typos.sh b/ci_v2/jobs/scripts/check_style/check_typos.sh new file mode 100755 index 00000000000..764101a6eac --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/check_typos.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Check for typos in code. + +ROOT_PATH="." + +#FIXME: check all (or almost all) repo +codespell \ + --skip "*generated*,*gperf*,*.bin,*.mrk*,*.idx,checksums.txt,*.dat,*.pyc,*.kate-swp,*obfuscateQueries.cpp,d3-*.js,*.min.js,*.sum,${ROOT_PATH}/utils/check-style/aspell-ignore" \ + --ignore-words "${ROOT_PATH}/utils/check-style/codespell-ignore-words.list" \ + --exclude-file "${ROOT_PATH}/utils/check-style/codespell-ignore-lines.list" \ + --quiet-level 2 \ + "$ROOT_PATH"/{src,base,programs,utils} \ + $@ | grep -P '.' \ + && echo -e "\nFound some typos in code.\nSee the files utils/check-style/codespell* if you want to add an exception." diff --git a/ci_v2/jobs/scripts/check_style/checks_to_refactor.sh b/ci_v2/jobs/scripts/check_style/checks_to_refactor.sh new file mode 100755 index 00000000000..ae4aae23c12 --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/checks_to_refactor.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +ROOT_PATH="." + +# Queries to system.query_log/system.query_thread_log should have current_database = currentDatabase() condition +# NOTE: it is not that accurate, but at least something. +tests_with_query_log=( $( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' | + xargs grep --with-filename -e system.query_log -e system.query_thread_log | cut -d: -f1 | sort -u +) ) +for test_case in "${tests_with_query_log[@]}"; do + grep -qE current_database.*currentDatabase "$test_case" || { + grep -qE 'current_database.*\$CLICKHOUSE_DATABASE' "$test_case" + } || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case" +done + +grep -iE 'SYSTEM STOP MERGES;?$' -R $ROOT_PATH/tests/queries && echo "Merges cannot be disabled globally in fast/stateful/stateless tests, because it will break concurrently running queries" + + +# Queries to: +tables_with_database_column=( + system.tables + system.parts + system.detached_parts + system.parts_columns + system.columns + system.projection_parts + system.mutations +) +# should have database = currentDatabase() condition +# +# NOTE: it is not that accuate, but at least something. +tests_with_database_column=( $( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' | + xargs grep --with-filename $(printf -- "-e %s " "${tables_with_database_column[@]}") | + grep -v -e ':--' -e ':#' | + cut -d: -f1 | sort -u +) ) +for test_case in "${tests_with_database_column[@]}"; do + grep -qE database.*currentDatabase "$test_case" || { + grep -qE 'database.*\$CLICKHOUSE_DATABASE' "$test_case" + } || { + # explicit database + grep -qE "database[ ]*=[ ]*'" "$test_case" + } || { + echo "Queries to ${tables_with_database_column[*]} does not have database = currentDatabase()/\$CLICKHOUSE_DATABASE condition in $test_case" + } +done + +# Queries with ReplicatedMergeTree +# NOTE: it is not that accuate, but at least something. +tests_with_replicated_merge_tree=( $( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' | + xargs grep --with-filename -e "Replicated.*MergeTree[ ]*(.*" | cut -d: -f1 | sort -u +) ) +for test_case in "${tests_with_replicated_merge_tree[@]}"; do + case "$test_case" in + *.gen.*) + ;; + *.sh) + test_case_zk_prefix="\(\$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX\|{database}\)" + grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)" + ;; + *.sql|*.sql.j2) + test_case_zk_prefix="\({database}\|currentDatabase()\|{uuid}\|{default_path_test}\)" + grep -q -e "Replicated.*MergeTree[ ]*(.*$test_case_zk_prefix" "$test_case" || echo "Replicated.*MergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)" + ;; + *.py) + # Right now there is not such tests anyway + echo "No ReplicatedMergeTree style check for *.py ($test_case)" + ;; + esac +done + +# The stateful directory should only contain the tests that depend on the test dataset (hits or visits). +find $ROOT_PATH/tests/queries/1_stateful -name '*.sql' -or -name '*.sh' | grep -v '00076_system_columns_bytes' | xargs -I{} bash -c 'grep -q -P "hits|visits" "{}" || echo "The test {} does not depend on the test dataset (hits or visits table) and should be located in the 0_stateless directory. You can also add an exception to the check-style script."' + +# Check for existence of __init__.py files +for i in "${ROOT_PATH}"/tests/integration/test_*; do FILE="${i}/__init__.py"; [ ! -f "${FILE}" ] && echo "${FILE} should exist for every integration test"; done + +# Check for executable bit on non-executable files +find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.j2' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -P '.' && echo "These files should not be executable." + +# Check for BOM +find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' | grep -P '.' && echo "Files should not have UTF-8 BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' | grep -P '.' && echo "Files should not have UTF-16LE BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' | grep -P '.' && echo "Files should not have UTF-16BE BOM" + +# Conflict markers +find $ROOT_PATH/{src,base,programs,utils,tests,docs,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | + xargs grep -P '^(<<<<<<<|=======|>>>>>>>)$' | grep -P '.' && echo "Conflict markers are found in files" + +# DOS/Windows newlines +find $ROOT_PATH/{base,src,programs,utils,docs} -name '*.md' -or -name '*.h' -or -name '*.cpp' -or -name '*.js' -or -name '*.py' -or -name '*.html' | xargs grep -l -P '\r$' && echo "^ Files contain DOS/Windows newlines (\r\n instead of \n)." + +# # workflows check +# act --list --directory="$ROOT_PATH" 1>/dev/null 2>&1 || act --list --directory="$ROOT_PATH" 2>&1 +# actionlint -ignore 'reusable workflow call.+' || : diff --git a/ci_v2/jobs/scripts/check_style/double_whitespaces.pl b/ci_v2/jobs/scripts/check_style/double_whitespaces.pl new file mode 100755 index 00000000000..daeddecbd27 --- /dev/null +++ b/ci_v2/jobs/scripts/check_style/double_whitespaces.pl @@ -0,0 +1,37 @@ +#!/usr/bin/perl + +use strict; + +# Find double whitespace such as "a, b, c" that looks very ugly and annoying. +# But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines. + +my $ret = 0; + +foreach my $file (@ARGV) +{ + my @array; + + open (FH,'<',$file); + while () + { + push @array, $_; + } + + for (my $i = 1; $i < $#array; ++$i) + { + if ($array[$i] =~ ',( {2,3})[^ /]') + { + # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl + + if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment + && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) + && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1, 2 } + { + print($file . ":" . ($i + 1) . $array[$i]); + $ret = 1; + } + } + } +} + +exit $ret; diff --git a/ci_v2/settings/definitions.py b/ci_v2/settings/definitions.py new file mode 100644 index 00000000000..87669cdcf25 --- /dev/null +++ b/ci_v2/settings/definitions.py @@ -0,0 +1,251 @@ +from praktika import Docker, Secret + +S3_BUCKET_NAME = "clickhouse-builds" +S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com" + + +class RunnerLabels: + CI_SERVICES = "ci_services" + CI_SERVICES_EBS = "ci_services_ebs" + + +BASE_BRANCH = "master" + +SECRETS = [ + Secret.Config( + name="dockerhub_robot_password", + type=Secret.Type.AWS_SSM_VAR, + ), + Secret.Config( + name="woolenwolf_gh_app.clickhouse-app-id", + type=Secret.Type.AWS_SSM_SECRET, + ), + Secret.Config( + name="woolenwolf_gh_app.clickhouse-app-key", + type=Secret.Type.AWS_SSM_SECRET, + ), +] + +DOCKERS = [ + # Docker.Config( + # name="clickhouse/binary-builder", + # path="./docker/packager/binary-builder", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/cctools", + # path="./docker/packager/cctools", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/test-old-centos", + # path="./docker/test/compatibility/centos", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/test-old-ubuntu", + # path="./docker/test/compatibility/ubuntu", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/test-util", + # path="./docker/test/util", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/integration-test", + # path="./docker/test/integration/base", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/fuzzer", + # path="./docker/test/fuzzer", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/performance-comparison", + # path="./docker/test/performance-comparison", + # arm64=True, + # amd64=True, + # depends_on=[], + # ), + # Docker.Config( + # name="clickhouse/fasttest", + # path="./docker/test/fasttest", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-util"], + # ), + # Docker.Config( + # name="clickhouse/test-base", + # path="./docker/test/base", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-util"], + # ), + # Docker.Config( + # name="clickhouse/clickbench", + # path="./docker/test/clickbench", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/keeper-jepsen-test", + # path="./docker/test/keeper-jepsen", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/server-jepsen-test", + # path="./docker/test/server-jepsen", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/sqllogic-test", + # path="./docker/test/sqllogic", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/sqltest", + # path="./docker/test/sqltest", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/stateless-test", + # path="./docker/test/stateless", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/stateful-test", + # path="./docker/test/stateful", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/stateless-test"], + # ), + # Docker.Config( + # name="clickhouse/stress-test", + # path="./docker/test/stress", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/stateful-test"], + # ), + # Docker.Config( + # name="clickhouse/unit-test", + # path="./docker/test/unit", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + # Docker.Config( + # name="clickhouse/integration-tests-runner", + # path="./docker/test/integration/runner", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), + Docker.Config( + name="clickhouse/style-test", + path="./ci_v2/docker/style-test", + platforms=Docker.Platforms.arm_amd, + depends_on=[], + ), + # Docker.Config( + # name="clickhouse/docs-builder", + # path="./docker/docs/builder", + # arm64=True, + # amd64=True, + # depends_on=["clickhouse/test-base"], + # ), +] + +# TODO: +# "docker/test/integration/s3_proxy": { +# "name": "clickhouse/s3-proxy", +# "dependent": [] +# }, +# "docker/test/integration/resolver": { +# "name": "clickhouse/python-bottle", +# "dependent": [] +# }, +# "docker/test/integration/helper_container": { +# "name": "clickhouse/integration-helper", +# "dependent": [] +# }, +# "docker/test/integration/mysql_golang_client": { +# "name": "clickhouse/mysql-golang-client", +# "dependent": [] +# }, +# "docker/test/integration/dotnet_client": { +# "name": "clickhouse/dotnet-client", +# "dependent": [] +# }, +# "docker/test/integration/mysql_java_client": { +# "name": "clickhouse/mysql-java-client", +# "dependent": [] +# }, +# "docker/test/integration/mysql_js_client": { +# "name": "clickhouse/mysql-js-client", +# "dependent": [] +# }, +# "docker/test/integration/mysql_php_client": { +# "name": "clickhouse/mysql-php-client", +# "dependent": [] +# }, +# "docker/test/integration/postgresql_java_client": { +# "name": "clickhouse/postgresql-java-client", +# "dependent": [] +# }, +# "docker/test/integration/kerberos_kdc": { +# "only_amd64": true, +# "name": "clickhouse/kerberos-kdc", +# "dependent": [] +# }, +# "docker/test/integration/kerberized_hadoop": { +# "only_amd64": true, +# "name": "clickhouse/kerberized-hadoop", +# "dependent": [] +# }, +# "docker/test/sqlancer": { +# "name": "clickhouse/sqlancer-test", +# "dependent": [] +# }, +# "docker/test/install/deb": { +# "name": "clickhouse/install-deb-test", +# "dependent": [] +# }, +# "docker/test/install/rpm": { +# "name": "clickhouse/install-rpm-test", +# "dependent": [] +# }, +# "docker/test/integration/nginx_dav": { +# "name": "clickhouse/nginx-dav", +# "dependent": [] +# } + + +class JobNames: + STYLE_CHECK = "Style Check" diff --git a/ci_v2/settings/settings.py b/ci_v2/settings/settings.py new file mode 100644 index 00000000000..153aab93506 --- /dev/null +++ b/ci_v2/settings/settings.py @@ -0,0 +1,20 @@ +from ci_v2.settings.definitions import ( + S3_BUCKET_HTTP_ENDPOINT, + S3_BUCKET_NAME, + RunnerLabels, +) + +S3_ARTIFACT_PATH = f"{S3_BUCKET_NAME}/artifacts" +CI_CONFIG_RUNS_ON = [RunnerLabels.CI_SERVICES] +DOCKER_BUILD_RUNS_ON = [RunnerLabels.CI_SERVICES_EBS] +CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" +HTML_S3_PATH = f"{S3_BUCKET_NAME}/reports" +S3_BUCKET_TO_HTTP_ENDPOINT = {S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT} + +DOCKERHUB_USERNAME = "robotclickhouse" +DOCKERHUB_SECRET = "dockerhub_robot_password" + +CI_DB_DB_NAME = "default" +CI_DB_TABLE_NAME = "checks" + +INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS = "" diff --git a/ci_v2/workflows/pull_request.py b/ci_v2/workflows/pull_request.py new file mode 100644 index 00000000000..226455c77f2 --- /dev/null +++ b/ci_v2/workflows/pull_request.py @@ -0,0 +1,44 @@ +from typing import List + +from ci_v2.settings.definitions import ( + BASE_BRANCH, + DOCKERS, + SECRETS, + JobNames, + RunnerLabels, +) +from praktika import Job, Workflow + +style_check_job = Job.Config( + name=JobNames.STYLE_CHECK, + runs_on=[RunnerLabels.CI_SERVICES], + command="python3 ./ci_v2/jobs/check_style.py", + run_in_docker="clickhouse/style-test", +) + +workflow = Workflow.Config( + name="PR", + event=Workflow.Event.PULL_REQUEST, + base_branches=[BASE_BRANCH], + jobs=[ + style_check_job, + ], + dockers=DOCKERS, + secrets=SECRETS, + enable_cache=True, + enable_report=True, + enable_merge_ready_status=True, +) + +WORKFLOWS = [ + workflow, +] # type: List[Workflow.Config] + + +if __name__ == "__main__": + # example: local job test inside praktika environment + from praktika.runner import Runner + + Runner.generate_dummy_environment(workflow, style_check_job) + + Runner().run(workflow, style_check_job) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 7191393533b..cc070ec72bd 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -69,6 +69,11 @@ set (SRCS_PROGRAM_OPTIONS "${LIBRARY_DIR}/libs/program_options/src/winmain.cpp" ) +# Always compile this file with the highest possible level of optimizations, even in Debug builds. +# Otherwise the client takes too long to start and SQL stateless tests (many options to parse) +# https://github.com/ClickHouse/ClickHouse/issues/65745 +set_source_files_properties(${SRCS_PROGRAM_OPTIONS} PROPERTIES COMPILE_FLAGS "-O3") + add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS}) add_library (boost::program_options ALIAS _boost_program_options) target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 191886981c3..59a424e25d0 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -164,15 +164,6 @@ target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) # Because our coverage callbacks call malloc, and recursive call of malloc could not work. target_compile_options(_jemalloc PRIVATE ${WITHOUT_COVERAGE_FLAGS_LIST}) -if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") - target_compile_definitions(_jemalloc PRIVATE - -DJEMALLOC_DEBUG=1 - # Usage examples: - # - MALLOC_CONF=log:. - # - MALLOC_CONF='log:core.malloc.exit|core.sallocx.entry|core.sdallocx.entry' - -DJEMALLOC_LOG=1) -endif () - target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) # jemalloc provides support two unwind flavors: diff --git a/docker/reqgenerator.py b/docker/reqgenerator.py index 6c1d89ac0ac..cbd25718dee 100644 --- a/docker/reqgenerator.py +++ b/docker/reqgenerator.py @@ -2,8 +2,8 @@ # To run this script you must install docker and piddeptree python package # -import subprocess import os +import subprocess import sys diff --git a/docker/test/fuzzer/generate-test-j2.py b/docker/test/fuzzer/generate-test-j2.py index 6fd37d6bd02..abbb3a06702 100755 --- a/docker/test/fuzzer/generate-test-j2.py +++ b/docker/test/fuzzer/generate-test-j2.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -from argparse import ArgumentParser -import os -import jinja2 import itertools +import os +from argparse import ArgumentParser + +import jinja2 def removesuffix(text, suffix): diff --git a/docker/test/integration/hive_server/http_api_server.py b/docker/test/integration/hive_server/http_api_server.py index 8a9d3da4846..b90e4c25eb0 100644 --- a/docker/test/integration/hive_server/http_api_server.py +++ b/docker/test/integration/hive_server/http_api_server.py @@ -1,7 +1,8 @@ +import datetime import os import subprocess -import datetime -from flask import Flask, flash, request, redirect, url_for + +from flask import Flask, flash, redirect, request, url_for def run_command(command, wait=False): diff --git a/docker/test/sqlancer/process_sqlancer_result.py b/docker/test/sqlancer/process_sqlancer_result.py index 06abb52abf8..9c87c5bbecb 100755 --- a/docker/test/sqlancer/process_sqlancer_result.py +++ b/docker/test/sqlancer/process_sqlancer_result.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import os -import logging import argparse import csv +import logging +import os def process_result(result_folder): diff --git a/docker/test/sqltest/test.py b/docker/test/sqltest/test.py index 5807ca79b02..8a8bf3cec4a 100755 --- a/docker/test/sqltest/test.py +++ b/docker/test/sqltest/test.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -import os -import yaml import html +import os import random import string -from clickhouse_driver import Client +import yaml +from clickhouse_driver import Client client = Client(host="localhost", port=9000) settings = { diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 4b11fb71230..64b6cd355ee 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -232,7 +232,7 @@ Below you can find some quick links which may be useful when writing code for Cl - [The code style guide](https://clickhouse.com/docs/en/development/style/). - [Adding third-party libraries](https://clickhouse.com/docs/en/development/contrib/#adding-third-party-libraries) - [Writing tests](https://clickhouse.com/docs/en/development/tests/) -- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3Ahacktoberfest) +- [List of open issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22) ## Writing Documentation {#writing-documentation} diff --git a/docs/en/getting-started/example-datasets/stackoverflow.md b/docs/en/getting-started/example-datasets/stackoverflow.md index e982a3c3dfc..defe157cc52 100644 --- a/docs/en/getting-started/example-datasets/stackoverflow.md +++ b/docs/en/getting-started/example-datasets/stackoverflow.md @@ -7,7 +7,7 @@ description: Analyzing Stack Overflow data with ClickHouse # Analyzing Stack Overflow data with ClickHouse -This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow. +This dataset contains every `Posts`, `Users`, `Votes`, `Comments`, `Badges`, `PostHistory`, and `PostLinks` that has occurred on Stack Overflow. Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months. @@ -159,7 +159,7 @@ INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentatio 0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.) ``` -### `PostLinks` +### PostLinks ```sql CREATE TABLE stackoverflow.postlinks @@ -178,7 +178,7 @@ INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documenta 0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.) ``` -### `PostHistory` +### PostHistory ```sql CREATE TABLE stackoverflow.posthistory diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 392b1831ce3..982a8f1dbf2 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -5683,6 +5683,24 @@ Enable `IF NOT EXISTS` for `CREATE` statement by default. If either this setting Default value: `false`. +## show_create_query_identifier_quoting_rule + +Define identifier quoting behavior of the show create query result: +- `when_necessary`: When the identifiers is one of `{"distinct", "all", "table"}`, or it can cause ambiguity: column names, dictionary attribute names. +- `always`: Always quote identifiers. +- `user_display`: When the identifiers is a keyword. + +Default value: `when_necessary`. + +## show_create_query_identifier_quoting_style + +Define identifier quoting style of the show create query result: +- `Backticks`: \`clickhouse\` style. +- `DoubleQuotes`: "postgres" style +- `BackticksMySQL`: \`mysql\` style, most same as `Backticks`, but it uses '``' to escape '`' + +Default value: `Backticks`. + ## mongodb_throw_on_unsupported_query If enabled, MongoDB tables will return an error when a MongoDB query can't be built. diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index fde39b3dc89..038ae3a0706 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -51,6 +51,40 @@ Calculates the MD5 from a string and returns the resulting set of bytes as Fixed If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead. If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))). +## RIPEMD160 + +Produces [RIPEMD-160](https://en.wikipedia.org/wiki/RIPEMD) hash value. + +**Syntax** + +```sql +RIPEMD160(input) +``` + +**Parameters** + +- `input`: Input string. [String](../data-types/string.md) + +**Returned value** + +- A 160-bit `RIPEMD-160` hash value of type [FixedString(20)](../data-types/fixedstring.md). + +**Example** + +Use the [hex](../functions/encoding-functions.md/#hex) function to represent the result as a hex-encoded string. + +Query: + +```sql +SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog')); +``` + +```response +┌─HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐ +│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │ +└───────────────────────────────────────────────────────────────┘ +``` + ## sipHash64 Produces a 64-bit [SipHash](https://en.wikipedia.org/wiki/SipHash) hash value. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 93437f71dcc..d17d05165e8 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2789,6 +2789,45 @@ Result: - [Custom Settings](../../operations/settings/index.md#custom_settings) +## getSettingOrDefault + +Returns the current value of a [custom setting](../../operations/settings/index.md#custom_settings) or returns the default value specified in the 2nd argument if the custom setting is not set in the current profile. + +**Syntax** + +```sql +getSettingOrDefault('custom_setting', default_value); +``` + +**Parameter** + +- `custom_setting` — The setting name. [String](../data-types/string.md). +- `default_value` — Value to return if custom_setting is not set. Value may be of any data type or Null. + +**Returned value** + +- The setting's current value or default_value if setting is not set. + +**Example** + +```sql +SELECT getSettingOrDefault('custom_undef1', 'my_value'); +SELECT getSettingOrDefault('custom_undef2', 100); +SELECT getSettingOrDefault('custom_undef3', NULL); +``` + +Result: + +``` +my_value +100 +NULL +``` + +**See Also** + +- [Custom Settings](../../operations/settings/index.md#custom_settings) + ## isDecimalOverflow Checks whether the [Decimal](../data-types/decimal.md) value is outside its precision or outside the specified precision. diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index f06f085a0ed..a56532e2ab0 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -12,10 +12,10 @@ Syntax: ``` sql ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]] [ON CLUSTER cluster_name] - [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}] + [NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} + [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] - [RESET AUTHENTICATION METHODS TO NEW] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 7428e6cd6ca..250cddb09dc 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -43,6 +43,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. +### With a Schema and Data Cloned from Another Table + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name CLONE AS [db2.]name2 [ENGINE = engine] +``` + +Creates a table with the same structure as another table. You can specify a different engine for the table. If the engine is not specified, the same engine will be used as for the `db2.name2` table. After the new table is created, all partitions from `db2.name2` are attached to it. In other words, the data of `db2.name2` is cloned into `db.table_name` upon creation. This query is equivalent to the following: + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine]; +ALTER TABLE [db.]table_name ATTACH PARTITION ALL FROM [db2].name2; +``` + ### From a Table Function ``` sql diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index ec160ea2663..a018e28306c 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -11,10 +11,10 @@ Syntax: ``` sql CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] + [NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} + [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] - [RESET AUTHENTICATION METHODS TO NEW] [IN access_storage_type] [DEFAULT ROLE role [,...]] [DEFAULT DATABASE database | NONE] diff --git a/docs/ru/sql-reference/statements/alter/user.md b/docs/ru/sql-reference/statements/alter/user.md index 86d5b8403b7..a00ea054863 100644 --- a/docs/ru/sql-reference/statements/alter/user.md +++ b/docs/ru/sql-reference/statements/alter/user.md @@ -13,8 +13,10 @@ sidebar_label: USER ``` sql ALTER USER [IF EXISTS] name1 [RENAME TO new_name |, name2 [,...]] [ON CLUSTER cluster_name] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}] + [NOT IDENTIFIED | RESET AUTHENTICATION METHODS TO NEW | {IDENTIFIED | ADD IDENTIFIED} {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} + [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [VALID UNTIL datetime] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md index 825ae021a17..660d8f55254 100644 --- a/docs/ru/sql-reference/statements/create/user.md +++ b/docs/ru/sql-reference/statements/create/user.md @@ -12,8 +12,11 @@ sidebar_label: "Пользователь" ``` sql CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [, name2 [,...]] [ON CLUSTER cluster_name] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'}] + [NOT IDENTIFIED | IDENTIFIED {[WITH {plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | WITH NO_PASSWORD | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']} + [, {[{plaintext_password | sha256_password | sha256_hash | ...}] BY {'password' | 'hash'}} | {ldap SERVER 'server_name'} | {...} | ... [,...]]] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [VALID UNTIL datetime] + [IN access_storage_type] [DEFAULT ROLE role [,...]] [DEFAULT DATABASE database | NONE] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] diff --git a/programs/disks/CommandRead.cpp b/programs/disks/CommandRead.cpp index 277e735f507..f4504504752 100644 --- a/programs/disks/CommandRead.cpp +++ b/programs/disks/CommandRead.cpp @@ -26,7 +26,7 @@ public: String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow(options, "path-from")); std::optional path_to = getValueFromCommandLineOptionsWithOptional(options, "path-to"); - auto in = disk.getDisk()->readFile(path_from); + auto in = disk.getDisk()->readFile(path_from, getReadSettings()); std::unique_ptr out = {}; if (path_to.has_value()) { diff --git a/programs/disks/CommandWrite.cpp b/programs/disks/CommandWrite.cpp index 9c82132e284..0497281b3c4 100644 --- a/programs/disks/CommandWrite.cpp +++ b/programs/disks/CommandWrite.cpp @@ -39,7 +39,7 @@ public: else { String relative_path_from = disk.getRelativeFromRoot(path_from.value()); - return disk.getDisk()->readFile(relative_path_from); + return disk.getDisk()->readFile(relative_path_from, getReadSettings()); } }(); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 04c02288858..82ebcc3340a 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -164,7 +164,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse std::string table_name = params.get("table_name"); LOG_TRACE(log, "DB name: '{}', table name: '{}'", db_name, table_name); - auto quoting_style = IdentifierQuotingStyle::None; + auto quoting_style = IdentifierQuotingStyle::Backticks; #if USE_ODBC quoting_style = getQuotingStyle(connection_handler); #endif diff --git a/programs/odbc-bridge/getIdentifierQuote.cpp b/programs/odbc-bridge/getIdentifierQuote.cpp index c0c833e5b8c..e96da91ecb9 100644 --- a/programs/odbc-bridge/getIdentifierQuote.cpp +++ b/programs/odbc-bridge/getIdentifierQuote.cpp @@ -38,7 +38,7 @@ IdentifierQuotingStyle getQuotingStyle(nanodbc::ConnectionHolderPtr connection) { auto identifier_quote = getIdentifierQuote(connection); if (identifier_quote.empty()) - return IdentifierQuotingStyle::None; + return IdentifierQuotingStyle::Backticks; else if (identifier_quote[0] == '`') return IdentifierQuotingStyle::Backticks; else if (identifier_quote[0] == '"') diff --git a/pyproject.toml b/pyproject.toml index 526dcc9a770..25228daf8f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ disable = ''' [tool.isort] profile = "black" -src_paths = ["src", "tests/ci", "tests/sqllogic"] +src_paths = ["src", "tests/ci", "tests/sqllogic", "tests/queries", "tests/integration"] [tool.black] required-version = 24 diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index e8ea21852b5..a24f8c5b2ad 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -30,6 +30,7 @@ namespace ErrorCodes { extern const int CANNOT_RESTORE_TABLE; extern const int ACCESS_ENTITY_ALREADY_EXISTS; + extern const int ACCESS_ENTITY_NOT_FOUND; extern const int LOGICAL_ERROR; } @@ -41,6 +42,7 @@ namespace { std::unordered_map entities; std::unordered_map> dependencies; + std::unordered_map dependents; BackupEntryPtr toBackupEntry() const { @@ -72,6 +74,24 @@ namespace } } + if (!dependents.empty()) + { + if (!dependencies.empty()) + writeText("\n", buf); + writeText("DEPENDENTS\n", buf); + for (const auto & [id, entity] : dependents) + { + writeText(id, buf); + writeChar('\t', buf); + writeText(entity->getTypeInfo().name, buf); + writeChar('\t', buf); + writeText(entity->getName(), buf); + writeChar('\n', buf); + writeText(serializeAccessEntity(*entity), buf); + writeChar('\n', buf); + } + } + return std::make_shared(buf.str()); } @@ -81,59 +101,71 @@ namespace { AccessEntitiesInBackup res; - bool dependencies_found = false; + bool reading_dependencies = false; + bool reading_dependents = false; while (!buf->eof()) { String line; readStringUntilNewlineInto(line, *buf); buf->ignore(); + if (line == "DEPENDENCIES") { - dependencies_found = true; - break; + reading_dependencies = true; + reading_dependents = false; + continue; + } + else if (line == "DEPENDENTS") + { + reading_dependents = true; + reading_dependencies = false; + continue; + } + else if (line.empty()) + { + continue; } - UUID id = parse(line.substr(0, line.find('\t'))); - line.clear(); + size_t separator1 = line.find('\t'); + size_t separator2 = line.find('\t', separator1 + 1); + if ((separator1 == String::npos) || (separator2 == String::npos)) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Separators not found in line {}", line); - String queries; - while (!buf->eof()) + UUID id = parse(line.substr(0, separator1)); + AccessEntityType type = AccessEntityTypeInfo::parseType(line.substr(separator1 + 1, separator2 - separator1 - 1)); + String name = line.substr(separator2 + 1); + + if (reading_dependencies) { - String query; - readStringUntilNewlineInto(query, *buf); - buf->ignore(); - if (query.empty()) - break; - if (!queries.empty()) - queries.append("\n"); - queries.append(query); + res.dependencies.emplace(id, std::pair{name, type}); } - - AccessEntityPtr entity = deserializeAccessEntity(queries); - res.entities.emplace(id, entity); - } - - if (dependencies_found) - { - while (!buf->eof()) + else { - String id_as_string; - readStringInto(id_as_string, *buf); - buf->ignore(); - UUID id = parse(id_as_string); + String queries; + while (!buf->eof()) + { + String query; + readStringUntilNewlineInto(query, *buf); + buf->ignore(); + if (query.empty()) + break; + if (!queries.empty()) + queries.append("\n"); + queries.append(query); + } - String type_as_string; - readStringInto(type_as_string, *buf); - buf->ignore(); - AccessEntityType type = AccessEntityTypeInfo::parseType(type_as_string); + AccessEntityPtr entity = deserializeAccessEntity(queries); - String name; - readStringInto(name, *buf); - buf->ignore(); + if (name != entity->getName()) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected name {} is specified for {}", name, entity->formatTypeWithName()); + if (type != entity->getType()) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Unexpected type {} is specified for {}", AccessEntityTypeInfo::get(type).name, entity->formatTypeWithName()); - if (!res.entities.contains(id)) - res.dependencies.emplace(id, std::pair{name, type}); + if (reading_dependents) + res.dependents.emplace(id, entity); + else + res.entities.emplace(id, entity); } } @@ -146,190 +178,59 @@ namespace } } }; - - std::vector findDependencies(const std::vector> & entities) - { - std::vector res; - for (const auto & entity : entities | boost::adaptors::map_values) - insertAtEnd(res, entity->findDependencies()); - - /// Remove duplicates in the list of dependencies (some entities can refer to other entities). - ::sort(res.begin(), res.end()); - res.erase(std::unique(res.begin(), res.end()), res.end()); - for (const auto & id : entities | boost::adaptors::map_keys) - { - auto it = std::lower_bound(res.begin(), res.end(), id); - if ((it != res.end()) && (*it == id)) - res.erase(it); - } - return res; - } - - std::unordered_map> readDependenciesNamesAndTypes(const std::vector & dependencies, const AccessControl & access_control) - { - std::unordered_map> res; - for (const auto & id : dependencies) - { - if (auto name_and_type = access_control.tryReadNameWithType(id)) - res.emplace(id, name_and_type.value()); - } - return res; - } - - /// Checks if new entities (which we're going to restore) already exist, - /// and either skips them or throws an exception depending on the restore settings. - void checkExistingEntities(std::vector> & entities, - std::unordered_map & old_to_new_id, - const AccessControl & access_control, - RestoreAccessCreationMode creation_mode) - { - if (creation_mode == RestoreAccessCreationMode::kReplace) - return; - - auto should_skip = [&](const std::pair & id_and_entity) - { - const auto & id = id_and_entity.first; - const auto & entity = *id_and_entity.second; - auto existing_id = access_control.find(entity.getType(), entity.getName()); - if (!existing_id) - { - return false; - } - else if (creation_mode == RestoreAccessCreationMode::kCreateIfNotExists) - { - old_to_new_id[id] = *existing_id; - return true; - } - else - { - throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", entity.formatTypeWithName()); - } - }; - - std::erase_if(entities, should_skip); - } - - /// If new entities (which we're going to restore) depend on other entities which are not going to be restored or not present in the backup - /// then we should try to replace those dependencies with already existing entities. - void resolveDependencies(const std::unordered_map> & dependencies, - std::unordered_map & old_to_new_ids, - const AccessControl & access_control, - bool allow_unresolved_dependencies) - { - for (const auto & [id, name_and_type] : dependencies) - { - std::optional new_id; - if (allow_unresolved_dependencies) - new_id = access_control.find(name_and_type.second, name_and_type.first); - else - new_id = access_control.getID(name_and_type.second, name_and_type.first); - if (new_id) - old_to_new_ids.emplace(id, *new_id); - } - } - - /// Generates random IDs for the new entities. - void generateRandomIDs(std::vector> & entities, std::unordered_map & old_to_new_ids) - { - Poco::UUIDGenerator generator; - for (auto & [id, entity] : entities) - { - UUID new_id; - generator.createRandom().copyTo(reinterpret_cast(&new_id)); - old_to_new_ids.emplace(id, new_id); - id = new_id; - } - } - - /// Updates dependencies of the new entities using a specified map. - void replaceDependencies(std::vector> & entities, - const std::unordered_map & old_to_new_ids) - { - for (auto & entity : entities | boost::adaptors::map_values) - IAccessEntity::replaceDependencies(entity, old_to_new_ids); - } - - AccessRightsElements getRequiredAccessToRestore(const std::vector> & entities) - { - AccessRightsElements res; - for (const auto & entity : entities | boost::adaptors::map_values) - { - auto entity_type = entity->getType(); - switch (entity_type) - { - case User::TYPE: - { - const auto & user = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_USER); - auto elements = user.access.getElements(); - for (auto & element : elements) - { - if (element.is_partial_revoke) - continue; - element.grant_option = true; - res.emplace_back(element); - } - if (!user.granted_roles.isEmpty()) - res.emplace_back(AccessType::ROLE_ADMIN); - break; - } - - case Role::TYPE: - { - const auto & role = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_ROLE); - auto elements = role.access.getElements(); - for (auto & element : elements) - { - if (element.is_partial_revoke) - continue; - element.grant_option = true; - res.emplace_back(element); - } - if (!role.granted_roles.isEmpty()) - res.emplace_back(AccessType::ROLE_ADMIN); - break; - } - - case SettingsProfile::TYPE: - { - res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE); - break; - } - - case RowPolicy::TYPE: - { - const auto & policy = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName()); - break; - } - - case Quota::TYPE: - { - res.emplace_back(AccessType::CREATE_QUOTA); - break; - } - - default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type)); - } - } - return res; - } } -std::pair makeBackupEntryForAccess( - const std::vector> & access_entities, - const String & data_path_in_backup, - size_t counter, - const AccessControl & access_control) +std::pair makeBackupEntryForAccessEntities( + const std::vector & entities_ids, + const std::unordered_map & all_entities, + bool write_dependents, + const String & data_path_in_backup) { - auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control); AccessEntitiesInBackup ab; - boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end())); - ab.dependencies = std::move(dependencies); - String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ... + + std::unordered_set entities_ids_set; + for (const auto & id : entities_ids) + entities_ids_set.emplace(id); + + for (const auto & id : entities_ids) + { + auto it = all_entities.find(id); + if (it != all_entities.end()) + { + AccessEntityPtr entity = it->second; + ab.entities.emplace(id, entity); + + auto dependencies = entity->findDependencies(); + for (const auto & dependency_id : dependencies) + { + if (!entities_ids_set.contains(dependency_id)) + { + auto it_dependency = all_entities.find(dependency_id); + if (it_dependency != all_entities.end()) + { + auto dependency_entity = it_dependency->second; + ab.dependencies.emplace(dependency_id, std::make_pair(dependency_entity->getName(), dependency_entity->getType())); + } + } + } + } + } + + if (write_dependents) + { + for (const auto & [id, possible_dependent] : all_entities) + { + if (!entities_ids_set.contains(id) && possible_dependent->hasDependencies(entities_ids_set)) + { + auto dependent = possible_dependent->clone(); + dependent->clearAllExceptDependencies(); + ab.dependents.emplace(id, dependent); + } + } + } + + String filename = fmt::format("access-{}.txt", UUIDHelpers::generateV4()); String file_path_in_backup = fs::path{data_path_in_backup} / filename; return {file_path_in_backup, ab.toBackupEntry()}; } @@ -339,61 +240,411 @@ AccessRestorerFromBackup::AccessRestorerFromBackup( const BackupPtr & backup_, const RestoreSettings & restore_settings_) : backup(backup_) , creation_mode(restore_settings_.create_access) - , allow_unresolved_dependencies(restore_settings_.allow_unresolved_access_dependencies) + , skip_unresolved_dependencies(restore_settings_.skip_unresolved_access_dependencies) + , update_dependents(restore_settings_.update_access_entities_dependents) + , log(getLogger("AccessRestorerFromBackup")) { } AccessRestorerFromBackup::~AccessRestorerFromBackup() = default; -void AccessRestorerFromBackup::addDataPath(const String & data_path) + +void AccessRestorerFromBackup::addDataPath(const String & data_path_in_backup) { - if (!data_paths.emplace(data_path).second) - return; + if (loaded) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities already loaded"); - fs::path data_path_in_backup_fs = data_path; - Strings filenames = backup->listFiles(data_path, /*recursive*/ false); - if (filenames.empty()) - return; - - for (const String & filename : filenames) - { - if (!filename.starts_with("access") || !filename.ends_with(".txt")) - throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"", - String{data_path_in_backup_fs / filename}); - } - - ::sort(filenames.begin(), filenames.end()); - - for (const String & filename : filenames) - { - String filepath_in_backup = data_path_in_backup_fs / filename; - auto read_buffer_from_backup = backup->readFile(filepath_in_backup); - auto ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup); - - boost::range::copy(ab.entities, std::back_inserter(entities)); - boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); - } - - for (const auto & id : entities | boost::adaptors::map_keys) - dependencies.erase(id); + if (std::find(data_paths_in_backup.begin(), data_paths_in_backup.end(), data_path_in_backup) == data_paths_in_backup.end()) + data_paths_in_backup.emplace_back(data_path_in_backup); } + +void AccessRestorerFromBackup::loadFromBackup() +{ + if (loaded) + return; + + /// Parse files "access*.txt" found in the added data paths in the backup. + for (size_t data_path_index = 0; data_path_index != data_paths_in_backup.size(); ++data_path_index) + { + const String & data_path_in_backup = data_paths_in_backup[data_path_index]; + + fs::path data_path_in_backup_fs = data_path_in_backup; + Strings filenames = backup->listFiles(data_path_in_backup_fs, /*recursive*/ false); + if (filenames.empty()) + continue; + + for (const String & filename : filenames) + { + if (!filename.starts_with("access") || !filename.ends_with(".txt")) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File name {} doesn't match the wildcard \"access*.txt\"", + String{data_path_in_backup_fs / filename}); + } + + for (const String & filename : filenames) + { + String filepath_in_backup = data_path_in_backup_fs / filename; + AccessEntitiesInBackup ab; + + try + { + auto read_buffer_from_backup = backup->readFile(filepath_in_backup); + ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup); + } + catch (Exception & e) + { + e.addMessage("While reading access entities from {} in backup", filepath_in_backup); + throw; + } + + for (const auto & [id, entity] : ab.entities) + { + auto it = entity_infos.find(id); + if (it == entity_infos.end()) + { + it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first; + } + EntityInfo & entity_info = it->second; + entity_info.entity = entity; + entity_info.restore = true; + entity_info.data_path_index = data_path_index; + } + + for (const auto & [id, name_and_type] : ab.dependencies) + { + auto it = entity_infos.find(id); + if (it == entity_infos.end()) + { + it = entity_infos.emplace(id, EntityInfo{.id = id, .name = name_and_type.first, .type = name_and_type.second}).first; + } + EntityInfo & entity_info = it->second; + entity_info.is_dependency = true; + } + + for (const auto & [id, entity] : ab.dependents) + { + auto it = entity_infos.find(id); + if (it == entity_infos.end()) + { + it = entity_infos.emplace(id, EntityInfo{.id = id, .name = entity->getName(), .type = entity->getType()}).first; + } + EntityInfo & entity_info = it->second; + if (!entity_info.restore) + entity_info.entity = entity; + } + } + } + + loaded = true; +} + + AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const { - return getRequiredAccessToRestore(entities); + if (!loaded) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded"); + + AccessRightsElements res; + for (const auto & [id, entity_info] : entity_infos) + { + if (!entity_info.restore) + continue; + const auto & entity = entity_info.entity; + auto entity_type = entity->getType(); + switch (entity_type) + { + case User::TYPE: + { + const auto & user = typeid_cast(*entity); + res.emplace_back(AccessType::CREATE_USER); + auto elements = user.access.getElements(); + for (auto & element : elements) + { + if (element.is_partial_revoke) + continue; + element.grant_option = true; + res.emplace_back(element); + } + if (!user.granted_roles.isEmpty()) + res.emplace_back(AccessType::ROLE_ADMIN); + break; + } + + case Role::TYPE: + { + const auto & role = typeid_cast(*entity); + res.emplace_back(AccessType::CREATE_ROLE); + auto elements = role.access.getElements(); + for (auto & element : elements) + { + if (element.is_partial_revoke) + continue; + element.grant_option = true; + res.emplace_back(element); + } + if (!role.granted_roles.isEmpty()) + res.emplace_back(AccessType::ROLE_ADMIN); + break; + } + + case SettingsProfile::TYPE: + { + res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE); + break; + } + + case RowPolicy::TYPE: + { + const auto & policy = typeid_cast(*entity); + res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName()); + break; + } + + case Quota::TYPE: + { + res.emplace_back(AccessType::CREATE_QUOTA); + break; + } + + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown type: {}", toString(entity_type)); + } + } + return res; } -std::vector> AccessRestorerFromBackup::getAccessEntities(const AccessControl & access_control) const + +void AccessRestorerFromBackup::generateRandomIDsAndResolveDependencies(const AccessControl & access_control) { - auto new_entities = entities; + if (ids_assigned) + return; + if (!loaded) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded"); + + /// Calculate `new_id` for each entity info. + /// Check which ones of the loaded access entities already exist. + /// Generate random UUIDs for access entities which we're going to restore if they don't exist. + for (auto & [id, entity_info] : entity_infos) + { + const String & name = entity_info.name; + auto type = entity_info.type; + + if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kReplace)) + { + entity_info.new_id = UUIDHelpers::generateV4(); + LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id); + continue; + } + + if (auto existing_id = access_control.find(type, name)) + { + if (entity_info.restore && (creation_mode == RestoreAccessCreationMode::kCreate)) + { + throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", + AccessEntityTypeInfo::get(type).formatEntityNameWithType(name)); + } + bool was_going_to_restore = entity_info.restore; + entity_info.new_id = *existing_id; + entity_info.restore = false; + LOG_TRACE(log, "{}: Found with UUID {}{}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *existing_id, + (was_going_to_restore ? ", will not restore" : "")); + } + else + { + if (entity_info.restore) + { + entity_info.new_id = UUIDHelpers::generateV4(); + LOG_TRACE(log, "{}: Generated new UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), *entity_info.new_id); + } + else if (skip_unresolved_dependencies) + { + LOG_TRACE(log, "{}: Not found, ignoring", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name)); + } + else + { + throw Exception(ErrorCodes::ACCESS_ENTITY_NOT_FOUND, "Cannot resolve {} while restoring from backup", + AccessEntityTypeInfo::get(type).formatEntityNameWithType(name)); + } + } + } + + /// Prepare map from old UUIDs to new UUIDs. + std::unordered_set ids_to_restore; std::unordered_map old_to_new_ids; - checkExistingEntities(new_entities, old_to_new_ids, access_control, creation_mode); - resolveDependencies(dependencies, old_to_new_ids, access_control, allow_unresolved_dependencies); - generateRandomIDs(new_entities, old_to_new_ids); - replaceDependencies(new_entities, old_to_new_ids); + std::unordered_set unresolved_ids; - return new_entities; + for (const auto & [id, entity_info] : entity_infos) + { + if (entity_info.restore) + ids_to_restore.insert(id); + + if (entity_info.new_id) + old_to_new_ids[id] = *entity_info.new_id; + else + unresolved_ids.insert(id); + } + + /// Calculate `is_dependent` for each entity info. + if (update_dependents) + { + for (auto & [id, entity_info] : entity_infos) + { + if (!entity_info.restore && entity_info.new_id && entity_info.entity && entity_info.entity->hasDependencies(ids_to_restore)) + entity_info.is_dependent = true; + } + } + + /// Remap the UUIDs of dependencies in the access entities we're going to restore. + for (auto & [id, entity_info] : entity_infos) + { + if (entity_info.restore || entity_info.is_dependent) + { + auto new_entity = entity_info.entity->clone(); + new_entity->replaceDependencies(old_to_new_ids); + new_entity->removeDependencies(unresolved_ids); + entity_info.entity = new_entity; + } + + if (entity_info.restore && data_path_with_entities_to_restore.empty()) + data_path_with_entities_to_restore = data_paths_in_backup[entity_info.data_path_index]; + } + + ids_assigned = true; +} + + +AccessEntitiesToRestore AccessRestorerFromBackup::getEntitiesToRestore(const String & data_path_in_backup) const +{ + if (!ids_assigned) + throw Exception(ErrorCodes::LOGICAL_ERROR, "IDs not assigned"); + + if (data_path_in_backup != data_path_with_entities_to_restore) + return {}; + + AccessEntitiesToRestore res; + res.new_entities.reserve(entity_infos.size()); + res.dependents.reserve(entity_infos.size()); + + for (const auto & [id, entity_info] : entity_infos) + { + if (entity_info.restore) + res.new_entities.emplace_back(*entity_info.new_id, entity_info.entity); + + if (entity_info.is_dependent) + res.dependents.emplace_back(AccessEntitiesToRestore::Dependent{*entity_info.new_id, entity_info.entity}); + } + + return res; +} + + +void restoreAccessEntitiesFromBackup( + IAccessStorage & destination_access_storage, + const AccessEntitiesToRestore & entities_to_restore, + const RestoreSettings & restore_settings) +{ + if (entities_to_restore.new_entities.empty()) + return; /// Nothing to restore. + + auto log = getLogger("AccessRestorerFromBackup"); + + bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + bool update_dependents = restore_settings.update_access_entities_dependents; + + std::unordered_set restored_ids; + std::unordered_map new_to_existing_ids; + AccessEntitiesToRestore::Dependents additional_dependents; + additional_dependents.reserve(entities_to_restore.new_entities.size()); + + for (const auto & [id, entity] : entities_to_restore.new_entities) + { + const String & name = entity->getName(); + auto type = entity->getType(); + LOG_TRACE(log, "{}: Adding with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), id); + + UUID existing_id; + if (destination_access_storage.insert(id, entity, replace_if_exists, throw_if_exists, &existing_id)) + { + LOG_TRACE(log, "{}: Added successfully", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name)); + restored_ids.emplace(id); + } + else + { + /// Couldn't insert `entity` because there is an existing entity with the same name. + LOG_TRACE(log, "{}: Not added because already exists with UUID {}", AccessEntityTypeInfo::get(type).formatEntityNameWithType(name), existing_id); + new_to_existing_ids[id] = existing_id; + if (update_dependents) + additional_dependents.emplace_back(AccessEntitiesToRestore::Dependent{existing_id, entity}); + } + } + + if (!new_to_existing_ids.empty()) + { + std::vector ids_to_update; + ids_to_update.reserve(restored_ids.size()); + boost::copy(restored_ids, std::inserter(ids_to_update, ids_to_update.end())); + + std::unordered_set new_ids; + boost::copy(new_to_existing_ids | boost::adaptors::map_keys, std::inserter(new_ids, new_ids.end())); + + /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed, + /// then we should correct those dependencies. + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr + { + if (!entity->hasDependencies(new_ids)) + return entity; + LOG_TRACE(log, "{}: Updating dependencies", entity->formatTypeWithName()); + auto res = entity->clone(); + res->replaceDependencies(new_to_existing_ids); + return res; + }; + + /// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here. + destination_access_storage.tryUpdate(ids_to_update, update_func); + } + + auto do_update_dependents = [&](const AccessEntitiesToRestore::Dependents & dependents) + { + if (dependents.empty()) + return; + + std::vector ids_to_update; + ids_to_update.reserve(dependents.size()); + std::unordered_map id_to_source; + + for (const auto & dependent : dependents) + { + const UUID & id = dependent.existing_id; + if (!destination_access_storage.isReadOnly(id)) + { + ids_to_update.emplace_back(id); + auto modified_source = dependent.source->clone(); + modified_source->replaceDependencies(new_to_existing_ids); + id_to_source[id] = modified_source; + } + } + + /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed, + /// then we should correct those dependencies. + auto update_func = [&](const AccessEntityPtr & entity, const UUID & existing_id) -> AccessEntityPtr + { + const auto & source = *id_to_source.at(existing_id); + if (!source.hasDependencies(restored_ids)) + return entity; + LOG_TRACE(log, "{}: Updating dependent", entity->formatTypeWithName()); + auto res = entity->clone(); + res->copyDependenciesFrom(source, restored_ids); + return res; + }; + + /// It's totally ok if some UUIDs from `ids_to_update` don't exist anymore, that's why we use tryUpdate() here. + destination_access_storage.tryUpdate(ids_to_update, update_func); + }; + + do_update_dependents(entities_to_restore.dependents); + do_update_dependents(additional_dependents); } } diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 51a1112e5d5..d5133b3cb9f 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include namespace DB @@ -12,6 +12,7 @@ enum class AccessEntityType : uint8_t; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; class AccessRightsElements; +class IAccessStorage; class IBackup; using BackupPtr = std::shared_ptr; class IBackupEntry; @@ -20,15 +21,42 @@ struct RestoreSettings; enum class RestoreAccessCreationMode : uint8_t; -/// Makes a backup of access entities of a specified type. -std::pair makeBackupEntryForAccess( - const std::vector> & access_entities, - const String & data_path_in_backup, - size_t counter, - const AccessControl & access_control); +/// Makes a backup entry for of a set of access entities. +std::pair makeBackupEntryForAccessEntities( + const std::vector & entities_ids, + const std::unordered_map & all_entities, + bool write_dependents, + const String & data_path_in_backup); +struct AccessEntitiesToRestore +{ + /// Access entities loaded from backup with new randomly generated UUIDs. + std::vector> new_entities; + + /// Dependents are access entities which exist already and they should be updated after restoring. + /// For example, if there were a role granted to a user: `CREATE USER user1; CREATE ROLE role1; GRANT role1 TO user1`, + /// and we're restoring only role `role1` because user `user1` already exists, + /// then user `user1` should be modified after restoring role `role1` to add this grant `GRANT role1 TO user1`. + struct Dependent + { + /// UUID of an existing access entities. + UUID existing_id; + + /// Source access entity from backup to copy dependencies from. + AccessEntityPtr source; + }; + using Dependents = std::vector; + Dependents dependents; +}; /// Restores access entities from a backup. +void restoreAccessEntitiesFromBackup( + IAccessStorage & access_storage, + const AccessEntitiesToRestore & entities_to_restore, + const RestoreSettings & restore_settings); + + +/// Loads access entities from a backup and prepares them for insertion into an access storage. class AccessRestorerFromBackup { public: @@ -36,21 +64,75 @@ public: ~AccessRestorerFromBackup(); /// Adds a data path to loads access entities from. - void addDataPath(const String & data_path); + void addDataPath(const String & data_path_in_backup); + + /// Loads access entities from the backup. + void loadFromBackup(); /// Checks that the current user can do restoring. + /// Function loadFromBackup() must be called before that. AccessRightsElements getRequiredAccess() const; - /// Inserts all access entities loaded from all the paths added by addDataPath(). - std::vector> getAccessEntities(const AccessControl & access_control) const; + /// Generates random IDs for access entities we're restoring to insert them into an access storage; + /// and finds IDs of existing access entities which are used as dependencies. + void generateRandomIDsAndResolveDependencies(const AccessControl & access_control); + + /// Returns access entities loaded from backup and prepared for insertion into an access storage. + /// Both functions loadFromBackup() and generateRandomIDsAndResolveDependencies() must be called before that. + AccessEntitiesToRestore getEntitiesToRestore(const String & data_path_in_backup) const; private: - BackupPtr backup; - RestoreAccessCreationMode creation_mode; - bool allow_unresolved_dependencies = false; - std::vector> entities; - std::unordered_map> dependencies; - std::unordered_set data_paths; + const BackupPtr backup; + const RestoreAccessCreationMode creation_mode; + const bool skip_unresolved_dependencies; + const bool update_dependents; + const LoggerPtr log; + + /// Whether loadFromBackup() finished. + bool loaded = false; + + /// Whether generateRandomIDsAndResolveDependencies() finished. + bool ids_assigned = false; + + Strings data_paths_in_backup; + String data_path_with_entities_to_restore; + + /// Information about an access entity loaded from the backup. + struct EntityInfo + { + UUID id; + String name; + AccessEntityType type; + + AccessEntityPtr entity = nullptr; /// Can be nullptr if `restore=false`. + + /// Index in `data_paths_in_backup`. + size_t data_path_index = 0; + + /// Whether we're going to restore this entity. + /// For example, + /// in case of `RESTORE TABLE system.roles` this flag is true for all the roles loaded from the backup, and + /// in case of `RESTORE ALL` this flag is always true. + bool restore = false; + + /// Whether this entity info was added as a dependency of another entity which we're going to restore. + /// For example, if we're going to restore the following user: `CREATE USER user1 DEFAULT ROLE role1, role2 SETTINGS PROFILE profile1, profile2` + /// then `restore=true` for `user1` and `is_dependency=true` for `role1`, `role2`, `profile1`, `profile2`. + /// Flags `restore` and `is_dependency` both can be set at the same time. + bool is_dependency = false; + + /// Whether this entity info is a dependent of another entity which we're going to restore. + /// For example, if we're going to restore role `role1` and there is also the following user stored in the backup: + /// `CREATE USER user1 DEFAULT ROLE role1`, then `is_dependent=true` for `user1`. + /// This flags is set by generateRandomIDsAndResolveDependencies(). + bool is_dependent = false; + + /// New UUID for this entity - either randomly generated or copied from an existing entity. + /// This UUID is assigned by generateRandomIDsAndResolveDependencies(). + std::optional new_id = std::nullopt; + }; + + std::unordered_map entity_infos; }; } diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index ec513f0692d..8e656372637 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -629,9 +629,9 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po } } -void AccessControl::restoreFromBackup(RestorerFromBackup & restorer) +void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) { - MultipleAccessStorage::restoreFromBackup(restorer); + MultipleAccessStorage::restoreFromBackup(restorer, data_path_in_backup); changes_notifier->sendNotifications(); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 0c3bb9352f0..cc1b7b2ca0d 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -124,7 +124,7 @@ public: AuthResult authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const String & forwarded_address) const; /// Makes a backup of access entities. - void restoreFromBackup(RestorerFromBackup & restorer) override; + void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override; void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index f5416a11433..ccaec79526b 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -594,6 +594,14 @@ public: optimizeTree(); } + void makeDifference(const Node & other) + { + Node rhs = other; + makeDifferenceRec(*this, rhs); + flags -= other.flags; + optimizeTree(); + } + ProtoElements getElements() const { ProtoElements res; @@ -1109,6 +1117,32 @@ private: result.wildcard_grant = wildcard_grant || rhs.wildcard_grant; } + void makeDifferenceRec(Node & result, Node & rhs) + { + if (rhs.children) + { + for (auto & rhs_child : *rhs.children) + { + auto & result_child = result.getLeaf(rhs_child.node_name, rhs_child.level, !rhs_child.isLeaf()); + auto & lhs_child = getLeaf(rhs_child.node_name, rhs_child.level, !rhs_child.isLeaf()); + lhs_child.makeDifferenceRec(result_child, rhs_child); + } + } + + if (children) + { + for (auto & lhs_child : *children) + { + auto & result_child = result.getLeaf(lhs_child.node_name, lhs_child.level, !lhs_child.isLeaf()); + auto & rhs_child = rhs.getLeaf(lhs_child.node_name, lhs_child.level, !lhs_child.isLeaf()); + lhs_child.makeDifferenceRec(result_child, rhs_child); + } + } + + result.flags = flags - rhs.flags; + result.wildcard_grant = wildcard_grant || rhs.wildcard_grant; + } + void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed) { if (children) @@ -1581,6 +1615,22 @@ void AccessRights::makeIntersection(const AccessRights & other) } +void AccessRights::makeDifference(const AccessRights & other) +{ + auto helper = [](std::unique_ptr & root_node, const std::unique_ptr & other_root_node) + { + if (!root_node || !other_root_node) + return; + + root_node->makeDifference(*other_root_node); + if (!root_node->flags && !root_node->children) + root_node = nullptr; + }; + helper(root, other.root); + helper(root_with_grant_option, other.root_with_grant_option); +} + + void AccessRights::modifyFlags(const ModifyFlagsFunction & function) { if (!root) diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index 6d905767b65..ea15338f6f3 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -150,6 +150,9 @@ public: /// Makes an intersection of access rights. void makeIntersection(const AccessRights & other); + /// Makes a difference (relative complement) of access rights. + void makeDifference(const AccessRights & other); + /// Traverse the tree and modify each access flags. using ModifyFlagsFunction = std::functionisGranted(flags, args...)) - { - return access_denied(ErrorCodes::ACCESS_DENIED, - "{}: Not enough privileges. " - "The required privileges have been granted, but without grant option. " - "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION", - AccessRightsElement{new_flags}.toStringForAccessTypeSource()); - } - - return access_denied(ErrorCodes::ACCESS_DENIED, - "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}", - AccessRightsElement{new_flags}.toStringForAccessTypeSource() + (grant_option ? " WITH GRANT OPTION" : "")); + return access_denied_no_grant(new_flags); } return access_denied_no_grant(flags, args...); diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 046c532cf5c..a9c69661725 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -676,7 +676,7 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ if (!entry.entity) entry.entity = readAccessEntityFromDisk(id); auto old_entity = entry.entity; - auto new_entity = update_func(old_entity); + auto new_entity = update_func(old_entity, id); if (!new_entity->isTypeOf(old_entity->getType())) throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp index e1a23182cc0..76e77599492 100644 --- a/src/Access/GrantedRoles.cpp +++ b/src/Access/GrantedRoles.cpp @@ -176,6 +176,16 @@ std::vector GrantedRoles::findDependencies() const return res; } +bool GrantedRoles::hasDependencies(const std::unordered_set & ids) const +{ + for (const auto & role_id : roles) + { + if (ids.contains(role_id)) + return true; + } + return false; +} + void GrantedRoles::replaceDependencies(const std::unordered_map & old_to_new_ids) { std::vector new_ids; @@ -221,4 +231,56 @@ void GrantedRoles::replaceDependencies(const std::unordered_map & ol } } +void GrantedRoles::copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set & ids) +{ + bool found = false; + + for (const auto & role_id : src.roles) + { + if (ids.contains(role_id)) + { + roles.emplace(role_id); + found = true; + } + } + + if (found) + { + for (const auto & role_id : src.roles_with_admin_option) + { + if (ids.contains(role_id)) + roles_with_admin_option.emplace(role_id); + } + } +} + +void GrantedRoles::removeDependencies(const std::unordered_set & ids) +{ + bool found = false; + + for (auto it = roles.begin(); it != roles.end();) + { + if (ids.contains(*it)) + { + it = roles.erase(it); + found = true; + } + else + { + ++it; + } + } + + if (found) + { + for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();) + { + if (ids.contains(*it)) + it = roles_with_admin_option.erase(it); + else + ++it; + } + } +} + } diff --git a/src/Access/GrantedRoles.h b/src/Access/GrantedRoles.h index ac252822089..d92af0f6db1 100644 --- a/src/Access/GrantedRoles.h +++ b/src/Access/GrantedRoles.h @@ -58,7 +58,10 @@ public: friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); } std::vector findDependencies() const; + bool hasDependencies(const std::unordered_set & ids) const; void replaceDependencies(const std::unordered_map & old_to_new_ids); + void copyDependenciesFrom(const GrantedRoles & src, const std::unordered_set & ids); + void removeDependencies(const std::unordered_set & ids); private: boost::container::flat_set roles; diff --git a/src/Access/IAccessEntity.cpp b/src/Access/IAccessEntity.cpp index 9afa1b73597..5dc566fe456 100644 --- a/src/Access/IAccessEntity.cpp +++ b/src/Access/IAccessEntity.cpp @@ -9,28 +9,4 @@ bool IAccessEntity::equal(const IAccessEntity & other) const return (name == other.name) && (getType() == other.getType()); } -void IAccessEntity::replaceDependencies(std::shared_ptr & entity, const std::unordered_map & old_to_new_ids) -{ - if (old_to_new_ids.empty()) - return; - - bool need_replace_dependencies = false; - auto dependencies = entity->findDependencies(); - for (const auto & dependency : dependencies) - { - if (old_to_new_ids.contains(dependency)) - { - need_replace_dependencies = true; - break; - } - } - - if (!need_replace_dependencies) - return; - - auto new_entity = entity->clone(); - new_entity->replaceDependencies(old_to_new_ids); - entity = new_entity; -} - } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 2c2df7353c5..09c5eb6cf10 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -48,10 +48,13 @@ struct IAccessEntity /// Finds all dependencies. virtual std::vector findDependencies() const { return {}; } + virtual bool hasDependencies(const std::unordered_set & /* ids */) const { return false; } /// Replaces dependencies according to a specified map. - void replaceDependencies(const std::unordered_map & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); } - static void replaceDependencies(std::shared_ptr & entity, const std::unordered_map & old_to_new_ids); + virtual void replaceDependencies(const std::unordered_map & /* old_to_new_ids */) {} + virtual void copyDependenciesFrom(const IAccessEntity & /* src */, const std::unordered_set & /* ids */) {} + virtual void removeDependencies(const std::unordered_set & /* ids */) {} + virtual void clearAllExceptDependencies() {} /// Whether this access entity should be written to a backup. virtual bool isBackupAllowed() const { return false; } @@ -67,8 +70,6 @@ protected: { return std::make_shared(typeid_cast(*this)); } - - virtual void doReplaceDependencies(const std::unordered_map & /* old_to_new_ids */) {} }; using AccessEntityPtr = std::shared_ptr; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 29475461c45..3249d89ba87 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -4,8 +4,10 @@ #include #include #include -#include +#include +#include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +74,18 @@ std::vector IAccessStorage::find(AccessEntityType type, const Strings & na } +std::vector IAccessStorage::findAllImpl() const +{ + std::vector res; + for (auto type : collections::range(AccessEntityType::MAX)) + { + auto ids = findAllImpl(type); + res.insert(res.end(), ids.begin(), ids.end()); + } + return res; +} + + UUID IAccessStorage::getID(AccessEntityType type, const String & name) const { auto id = findImpl(type, name); @@ -598,67 +613,59 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c if (!isBackupAllowed()) throwBackupNotAllowed(); - auto entities = readAllWithIDs(type); - std::erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); - - if (entities.empty()) + auto entities_ids = findAll(type); + if (entities_ids.empty()) return; - auto backup_entry = makeBackupEntryForAccess( - entities, - data_path_in_backup, - backup_entries_collector.getAccessCounter(type), - backup_entries_collector.getContext()->getAccessControl()); + auto backup_entry_with_path = makeBackupEntryForAccessEntities( + entities_ids, + backup_entries_collector.getAllAccessEntities(), + backup_entries_collector.getBackupSettings().write_access_entities_dependents, + data_path_in_backup); - backup_entries_collector.addBackupEntry(backup_entry); + if (isReplicated()) + { + auto backup_coordination = backup_entries_collector.getBackupCoordination(); + auto replication_id = getReplicationID(); + backup_coordination->addReplicatedAccessFilePath(replication_id, type, backup_entry_with_path.first); + + backup_entries_collector.addPostTask( + [backup_entry = backup_entry_with_path.second, + replication_id, + type, + &backup_entries_collector, + backup_coordination] + { + for (const String & path : backup_coordination->getReplicatedAccessFilePaths(replication_id, type)) + backup_entries_collector.addBackupEntry(path, backup_entry); + }); + } + else + { + backup_entries_collector.addBackupEntry(backup_entry_with_path); + } } -void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) +void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - if (isReplicated() && !acquireReplicatedRestore(restorer)) - return; - - auto entities = restorer.getAccessEntitiesToRestore(); - if (entities.empty()) - return; - - auto create_access = restorer.getRestoreSettings().create_access; - bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); - - restorer.addDataRestoreTask([this, entities_to_restore = std::move(entities), replace_if_exists, throw_if_exists] mutable + if (isReplicated()) { - std::unordered_map new_to_existing_ids; - for (auto & [id, entity] : entities_to_restore) - { - UUID existing_entity_id; - if (!insert(id, entity, replace_if_exists, throw_if_exists, &existing_entity_id)) - { - /// Couldn't insert `entity` because there is an existing entity with the same name. - new_to_existing_ids[id] = existing_entity_id; - } - } + auto restore_coordination = restorer.getRestoreCoordination(); + if (!restore_coordination->acquireReplicatedAccessStorage(getReplicationID())) + return; + } - if (!new_to_existing_ids.empty()) + restorer.addDataRestoreTask( + [this, &restorer, data_path_in_backup] { - /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed, - /// then we should correct those dependencies. - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr - { - auto res = entity; - IAccessEntity::replaceDependencies(res, new_to_existing_ids); - return res; - }; - std::vector ids; - ids.reserve(entities_to_restore.size()); - boost::copy(entities_to_restore | boost::adaptors::map_keys, std::back_inserter(ids)); - tryUpdate(ids, update_func); - } - }); + auto entities_to_restore = restorer.getAccessEntitiesToRestore(data_path_in_backup); + const auto & restore_settings = restorer.getRestoreSettings(); + restoreAccessEntitiesFromBackup(*this, entities_to_restore, restore_settings); + }); } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index a8ac75075d3..84cbdd0a751 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -66,6 +66,7 @@ public: /// Returns true if this storage is replicated. virtual bool isReplicated() const { return false; } + virtual String getReplicationID() const { return ""; } /// Starts periodic reloading and updating of entities in this storage. virtual void startPeriodicReloading() {} @@ -90,8 +91,9 @@ public: /// Returns the identifiers of all the entities of a specified type contained in the storage. std::vector findAll(AccessEntityType type) const; - template - std::vector findAll() const { return findAll(EntityClassT::TYPE); } + /// Returns the identifiers of all the entities in the storage. + template + std::vector findAll() const; /// Searches for an entity with specified type and name. Returns std::nullopt if not found. std::optional find(AccessEntityType type, const String & name) const; @@ -148,7 +150,7 @@ public: std::optional> tryReadNameWithType(const UUID & id) const; /// Reads all entities and returns them with their IDs. - template + template std::vector>> readAllWithIDs() const; std::vector> readAllWithIDs(AccessEntityType type) const; @@ -180,7 +182,7 @@ public: /// Removes multiple entities from the storage. Returns the list of successfully dropped. std::vector tryRemove(const std::vector & ids); - using UpdateFunc = std::function; + using UpdateFunc = std::function; /// Updates an entity stored in the storage. Throws an exception if couldn't update. bool update(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists = true); @@ -214,11 +216,12 @@ public: /// Makes a backup of this access storage. virtual void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const; - virtual void restoreFromBackup(RestorerFromBackup & restorer); + virtual void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup); protected: virtual std::optional findImpl(AccessEntityType type, const String & name) const = 0; virtual std::vector findAllImpl(AccessEntityType type) const = 0; + virtual std::vector findAllImpl() const; virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0; virtual std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const; virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id); @@ -267,6 +270,16 @@ private: }; +template +std::vector IAccessStorage::findAll() const +{ + if constexpr (std::is_same_v) + return findAllImpl(); + else + return findAllImpl(EntityClassT::TYPE); +} + + template std::shared_ptr IAccessStorage::read(const UUID & id, bool throw_if_not_exists) const { diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 3576d38fbda..4fbe0242bd7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -163,7 +163,7 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, // Update the granted roles of the relevant users. if (!user_ids.empty()) { - auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_) -> AccessEntityPtr + auto update_func = [&role_id, &grant] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr { if (auto user = typeid_cast>(entity_)) { @@ -301,7 +301,7 @@ void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String if (it != external_role_hashes.end() && it->second == external_roles_hash) return; - auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_) -> AccessEntityPtr + auto update_func = [this, &external_roles, external_roles_hash] (const AccessEntityPtr & entity_, const UUID &) -> AccessEntityPtr { if (auto user = typeid_cast>(entity_)) { diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 3b5a987fc6e..e1743cdd6a4 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -204,7 +204,7 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat Entry & entry = it->second; auto old_entity = entry.entity; - auto new_entity = update_func(old_entity); + auto new_entity = update_func(old_entity, id); if (!new_entity->isTypeOf(old_entity->getType())) throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index f1da8359d48..d51a7e00511 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -416,7 +416,7 @@ bool MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat { if (auto old_entity = storage_for_updating->tryRead(id)) { - auto new_entity = update_func(old_entity); + auto new_entity = update_func(old_entity, id); if (new_entity->getName() != old_entity->getName()) { for (const auto & storage : *storages) @@ -508,7 +508,7 @@ void MultipleAccessStorage::backup(BackupEntriesCollector & backup_entries_colle throwBackupNotAllowed(); } -void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) +void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) { auto storages = getStoragesInternal(); @@ -516,7 +516,7 @@ void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (storage->isRestoreAllowed()) { - storage->restoreFromBackup(restorer); + storage->restoreFromBackup(restorer, data_path_in_backup); return; } } diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 352cc7f7457..06147a45ba7 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -59,7 +59,7 @@ public: bool isBackupAllowed() const override; bool isRestoreAllowed() const override; void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; - void restoreFromBackup(RestorerFromBackup & restorer) override; + void restoreFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup) override; bool containsStorage(std::string_view storage_type) const; protected: diff --git a/src/Access/Quota.cpp b/src/Access/Quota.cpp index ead5f77ce57..3d4bff1ccba 100644 --- a/src/Access/Quota.cpp +++ b/src/Access/Quota.cpp @@ -24,9 +24,33 @@ std::vector Quota::findDependencies() const return to_roles.findDependencies(); } -void Quota::doReplaceDependencies(const std::unordered_map & old_to_new_ids) +bool Quota::hasDependencies(const std::unordered_set & ids) const +{ + return to_roles.hasDependencies(ids); +} + +void Quota::replaceDependencies(const std::unordered_map & old_to_new_ids) { to_roles.replaceDependencies(old_to_new_ids); } +void Quota::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) +{ + if (getType() != src.getType()) + return; + const auto & src_quota = typeid_cast(src); + to_roles.copyDependenciesFrom(src_quota.to_roles, ids); +} + +void Quota::removeDependencies(const std::unordered_set & ids) +{ + to_roles.removeDependencies(ids); +} + +void Quota::clearAllExceptDependencies() +{ + all_limits.clear(); + key_type = QuotaKeyType::NONE; +} + } diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 69ec2eb53a5..2281714eb13 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -47,7 +47,12 @@ struct Quota : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; + bool hasDependencies(const std::unordered_set & ids) const override; + void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) override; + void removeDependencies(const std::unordered_set & ids) override; + void clearAllExceptDependencies() override; + bool isBackupAllowed() const override { return true; } }; diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 9039a3b98b7..f0cb8a5c9f0 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -4,10 +4,6 @@ #include #include #include -#include -#include -#include -#include #include #include #include @@ -359,7 +355,7 @@ bool ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke } const AccessEntityPtr old_entity = deserializeAccessEntity(old_entity_definition, entity_path); - const AccessEntityPtr new_entity = update_func(old_entity); + const AccessEntityPtr new_entity = update_func(old_entity, id); if (!new_entity->isTypeOf(old_entity->getType())) throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); @@ -684,44 +680,4 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if return memory_storage.read(id, throw_if_not_exists); } - -void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const -{ - if (!isBackupAllowed()) - throwBackupNotAllowed(); - - auto entities = readAllWithIDs(type); - std::erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); - - if (entities.empty()) - return; - - auto backup_entry_with_path = makeBackupEntryForAccess( - entities, - data_path_in_backup, - backup_entries_collector.getAccessCounter(type), - backup_entries_collector.getContext()->getAccessControl()); - - auto backup_coordination = backup_entries_collector.getBackupCoordination(); - backup_coordination->addReplicatedAccessFilePath(zookeeper_path, type, backup_entry_with_path.first); - - backup_entries_collector.addPostTask( - [backup_entry = backup_entry_with_path.second, - my_zookeeper_path = zookeeper_path, - type, - &backup_entries_collector, - backup_coordination] - { - for (const String & path : backup_coordination->getReplicatedAccessFilePaths(my_zookeeper_path, type)) - backup_entries_collector.addBackupEntry(path, backup_entry); - }); -} - - -bool ReplicatedAccessStorage::acquireReplicatedRestore(RestorerFromBackup & restorer) const -{ - auto restore_coordination = restorer.getRestoreCoordination(); - return restore_coordination->acquireReplicatedAccessStorage(zookeeper_path); -} - } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 528dbb31c24..e46e854d414 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -26,7 +26,9 @@ public: void shutdown() override; const char * getStorageType() const override { return STORAGE_TYPE; } + bool isReplicated() const override { return true; } + String getReplicationID() const override { return zookeeper_path; } void startPeriodicReloading() override { startWatchingThread(); } void stopPeriodicReloading() override { stopWatchingThread(); } @@ -35,7 +37,6 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; private: String zookeeper_path; @@ -80,7 +81,6 @@ private: std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; - bool acquireReplicatedRestore(RestorerFromBackup & restorer) const override; mutable std::mutex mutex; MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex); diff --git a/src/Access/Role.cpp b/src/Access/Role.cpp index f6250594103..4140daa0fd9 100644 --- a/src/Access/Role.cpp +++ b/src/Access/Role.cpp @@ -21,10 +21,36 @@ std::vector Role::findDependencies() const return res; } -void Role::doReplaceDependencies(const std::unordered_map & old_to_new_ids) +bool Role::hasDependencies(const std::unordered_set & ids) const +{ + return granted_roles.hasDependencies(ids) || settings.hasDependencies(ids); +} + +void Role::replaceDependencies(const std::unordered_map & old_to_new_ids) { granted_roles.replaceDependencies(old_to_new_ids); settings.replaceDependencies(old_to_new_ids); } +void Role::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) +{ + if (getType() != src.getType()) + return; + const auto & src_role = typeid_cast(src); + granted_roles.copyDependenciesFrom(src_role.granted_roles, ids); + settings.copyDependenciesFrom(src_role.settings, ids); +} + +void Role::removeDependencies(const std::unordered_set & ids) +{ + granted_roles.removeDependencies(ids); + settings.removeDependencies(ids); +} + +void Role::clearAllExceptDependencies() +{ + access = {}; + settings.removeSettingsKeepProfiles(); +} + } diff --git a/src/Access/Role.h b/src/Access/Role.h index c7f98585a6c..7b2450adfbe 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -21,7 +21,12 @@ struct Role : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; + bool hasDependencies(const std::unordered_set & ids) const override; + void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) override; + void removeDependencies(const std::unordered_set & ids) override; + void clearAllExceptDependencies() override; + bool isBackupAllowed() const override { return settings.isBackupAllowed(); } }; diff --git a/src/Access/RolesOrUsersSet.cpp b/src/Access/RolesOrUsersSet.cpp index c026ae42f76..3568b60e44d 100644 --- a/src/Access/RolesOrUsersSet.cpp +++ b/src/Access/RolesOrUsersSet.cpp @@ -295,6 +295,23 @@ std::vector RolesOrUsersSet::findDependencies() const return res; } +bool RolesOrUsersSet::hasDependencies(const std::unordered_set & dependencies_ids) const +{ + for (const auto & id : ids) + { + if (dependencies_ids.contains(id)) + return true; + } + + for (const auto & id : except_ids) + { + if (dependencies_ids.contains(id)) + return true; + } + + return false; +} + void RolesOrUsersSet::replaceDependencies(const std::unordered_map & old_to_new_ids) { std::vector new_ids; @@ -337,4 +354,41 @@ void RolesOrUsersSet::replaceDependencies(const std::unordered_map & boost::range::copy(new_ids, std::inserter(except_ids, except_ids.end())); } +void RolesOrUsersSet::copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set & dependencies_ids) +{ + if (all != src.all) + return; + + for (const auto & id : src.ids) + { + if (dependencies_ids.contains(id)) + ids.emplace(id); + } + + for (const auto & id : src.except_ids) + { + if (dependencies_ids.contains(id)) + except_ids.emplace(id); + } +} + +void RolesOrUsersSet::removeDependencies(const std::unordered_set & dependencies_ids) +{ + for (auto it = ids.begin(); it != ids.end();) + { + if (dependencies_ids.contains(*it)) + it = ids.erase(it); + else + ++it; + } + + for (auto it = except_ids.begin(); it != except_ids.end();) + { + if (dependencies_ids.contains(*it)) + except_ids.erase(it); + else + ++it; + } +} + } diff --git a/src/Access/RolesOrUsersSet.h b/src/Access/RolesOrUsersSet.h index 29e499bc81b..93d94708657 100644 --- a/src/Access/RolesOrUsersSet.h +++ b/src/Access/RolesOrUsersSet.h @@ -64,7 +64,10 @@ struct RolesOrUsersSet friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); } std::vector findDependencies() const; + bool hasDependencies(const std::unordered_set & dependencies_ids) const; void replaceDependencies(const std::unordered_map & old_to_new_ids); + void copyDependenciesFrom(const RolesOrUsersSet & src, const std::unordered_set & dependencies_ids); + void removeDependencies(const std::unordered_set & dependencies_ids); bool all = false; boost::container::flat_set ids; diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index 8724d0f513c..5b9d8521d9a 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -63,9 +63,33 @@ std::vector RowPolicy::findDependencies() const return to_roles.findDependencies(); } -void RowPolicy::doReplaceDependencies(const std::unordered_map & old_to_new_ids) +bool RowPolicy::hasDependencies(const std::unordered_set & ids) const +{ + return to_roles.hasDependencies(ids); +} + +void RowPolicy::replaceDependencies(const std::unordered_map & old_to_new_ids) { to_roles.replaceDependencies(old_to_new_ids); } +void RowPolicy::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) +{ + if (getType() != src.getType()) + return; + const auto & src_policy = typeid_cast(src); + to_roles.copyDependenciesFrom(src_policy.to_roles, ids); +} + +void RowPolicy::removeDependencies(const std::unordered_set & ids) +{ + to_roles.removeDependencies(ids); +} + +void RowPolicy::clearAllExceptDependencies() +{ + for (auto & filter : filters) + filter = {}; +} + } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 5cfe85c186a..3ce96b24823 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -50,7 +50,12 @@ struct RowPolicy : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; + bool hasDependencies(const std::unordered_set & ids) const override; + void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) override; + void removeDependencies(const std::unordered_set & ids) override; + void clearAllExceptDependencies() override; + bool isBackupAllowed() const override { return true; } /// Which roles or users should use this row policy. diff --git a/src/Access/SettingsProfile.cpp b/src/Access/SettingsProfile.cpp index 632bd97fbf5..8f863b29e86 100644 --- a/src/Access/SettingsProfile.cpp +++ b/src/Access/SettingsProfile.cpp @@ -21,10 +21,35 @@ std::vector SettingsProfile::findDependencies() const return res; } -void SettingsProfile::doReplaceDependencies(const std::unordered_map & old_to_new_ids) +bool SettingsProfile::hasDependencies(const std::unordered_set & ids) const +{ + return elements.hasDependencies(ids) || to_roles.hasDependencies(ids); +} + +void SettingsProfile::replaceDependencies(const std::unordered_map & old_to_new_ids) { elements.replaceDependencies(old_to_new_ids); to_roles.replaceDependencies(old_to_new_ids); } +void SettingsProfile::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) +{ + if (getType() != src.getType()) + return; + const auto & src_profile = typeid_cast(src); + elements.copyDependenciesFrom(src_profile.elements, ids); + to_roles.copyDependenciesFrom(src_profile.to_roles, ids); +} + +void SettingsProfile::removeDependencies(const std::unordered_set & ids) +{ + elements.removeDependencies(ids); + to_roles.removeDependencies(ids); +} + +void SettingsProfile::clearAllExceptDependencies() +{ + elements.removeSettingsKeepProfiles(); +} + } diff --git a/src/Access/SettingsProfile.h b/src/Access/SettingsProfile.h index 6bcaf6fef30..4e70a3d6727 100644 --- a/src/Access/SettingsProfile.h +++ b/src/Access/SettingsProfile.h @@ -22,7 +22,12 @@ struct SettingsProfile : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; + bool hasDependencies(const std::unordered_set & ids) const override; + void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) override; + void removeDependencies(const std::unordered_set & ids) override; + void clearAllExceptDependencies() override; + bool isBackupAllowed() const override { return elements.isBackupAllowed(); } }; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 156ffcf197d..1de61771f93 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -158,6 +158,18 @@ std::vector SettingsProfileElements::findDependencies() const } +bool SettingsProfileElements::hasDependencies(const std::unordered_set & ids) const +{ + std::vector res; + for (const auto & element : *this) + { + if (element.parent_profile && ids.contains(*element.parent_profile)) + return true; + } + return false; +} + + void SettingsProfileElements::replaceDependencies(const std::unordered_map & old_to_new_ids) { for (auto & element : *this) @@ -176,6 +188,38 @@ void SettingsProfileElements::replaceDependencies(const std::unordered_map & ids) +{ + SettingsProfileElements new_elements; + for (const auto & element : src) + { + if (element.parent_profile && ids.contains(*element.parent_profile)) + { + SettingsProfileElement new_element; + new_element.parent_profile = *element.parent_profile; + new_elements.emplace_back(new_element); + } + } + insert(begin(), new_elements.begin(), new_elements.end()); +} + + +void SettingsProfileElements::removeDependencies(const std::unordered_set & ids) +{ + std::erase_if( + *this, [&](const SettingsProfileElement & element) { return element.parent_profile && ids.contains(*element.parent_profile); }); +} + + +void SettingsProfileElements::removeSettingsKeepProfiles() +{ + for (auto & element : *this) + element.setting_name.clear(); + + std::erase_if(*this, [&](const SettingsProfileElement & element) { return element.setting_name.empty() && !element.parent_profile; }); +} + + void SettingsProfileElements::merge(const SettingsProfileElements & other) { insert(end(), other.begin(), other.end()); diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index 7078f565295..ea42b823ce1 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -63,7 +63,12 @@ public: std::shared_ptr toASTWithNames(const AccessControl & access_control) const; std::vector findDependencies() const; + bool hasDependencies(const std::unordered_set & ids) const; void replaceDependencies(const std::unordered_map & old_to_new_ids); + void copyDependenciesFrom(const SettingsProfileElements & src, const std::unordered_set & ids); + void removeDependencies(const std::unordered_set & ids); + + void removeSettingsKeepProfiles(); void merge(const SettingsProfileElements & other); diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 2052527f4ae..887abc213f9 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -49,7 +49,12 @@ std::vector User::findDependencies() const return res; } -void User::doReplaceDependencies(const std::unordered_map & old_to_new_ids) +bool User::hasDependencies(const std::unordered_set & ids) const +{ + return default_roles.hasDependencies(ids) || granted_roles.hasDependencies(ids) || grantees.hasDependencies(ids) || settings.hasDependencies(ids); +} + +void User::replaceDependencies(const std::unordered_map & old_to_new_ids) { default_roles.replaceDependencies(old_to_new_ids); granted_roles.replaceDependencies(old_to_new_ids); @@ -57,4 +62,33 @@ void User::doReplaceDependencies(const std::unordered_map & old_to_n settings.replaceDependencies(old_to_new_ids); } +void User::copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) +{ + if (getType() != src.getType()) + return; + const auto & src_user = typeid_cast(src); + default_roles.copyDependenciesFrom(src_user.default_roles, ids); + granted_roles.copyDependenciesFrom(src_user.granted_roles, ids); + grantees.copyDependenciesFrom(src_user.grantees, ids); + settings.copyDependenciesFrom(src_user.settings, ids); +} + +void User::removeDependencies(const std::unordered_set & ids) +{ + default_roles.removeDependencies(ids); + granted_roles.removeDependencies(ids); + grantees.removeDependencies(ids); + settings.removeDependencies(ids); +} + +void User::clearAllExceptDependencies() +{ + authentication_methods.clear(); + allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; + access = {}; + settings.removeSettingsKeepProfiles(); + default_database = {}; + valid_until = 0; +} + } diff --git a/src/Access/User.h b/src/Access/User.h index 7f91c1e3756..03d62bf2277 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -32,7 +32,12 @@ struct User : public IAccessEntity void setName(const String & name_) override; std::vector findDependencies() const override; - void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; + bool hasDependencies(const std::unordered_set & ids) const override; + void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void copyDependenciesFrom(const IAccessEntity & src, const std::unordered_set & ids) override; + void removeDependencies(const std::unordered_set & ids) override; + void clearAllExceptDependencies() override; + bool isBackupAllowed() const override { return settings.isBackupAllowed(); } }; diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp index e5b62bc94a3..902fc949840 100644 --- a/src/Access/tests/gtest_access_rights_ops.cpp +++ b/src/Access/tests/gtest_access_rights_ops.cpp @@ -445,6 +445,48 @@ TEST(AccessRights, Intersection) ASSERT_EQ(lhs.toString(), "GRANT SELECT ON toaster.*"); } +TEST(AccessRights, Difference) +{ + AccessRights lhs, rhs; + lhs.grant(AccessType::SELECT); + rhs.grant(AccessType::SELECT); + rhs.revoke(AccessType::SELECT, "system"); + lhs.makeDifference(rhs); + ASSERT_EQ(lhs.toString(), "GRANT SELECT ON system.*"); + + lhs = {}; + rhs = {}; + lhs.grantWildcard(AccessType::SELECT, "toast"); + rhs.grant(AccessType::SELECT); + rhs.revoke(AccessType::SELECT, "toaster"); + lhs.makeDifference(rhs); + ASSERT_EQ(lhs.toString(), "GRANT SELECT ON toaster.*"); + + lhs = {}; + rhs = {}; + lhs.grantWildcard(AccessType::SELECT, "toast"); + lhs.grant(AccessType::CREATE_TABLE, "jam"); + auto lhs_old = lhs; + lhs.makeDifference(rhs); + ASSERT_EQ(lhs, lhs_old); + + lhs = {}; + rhs = {}; + lhs.grant(AccessType::SELECT, "toast"); + rhs.grant(AccessType::CREATE_TABLE, "jam"); + lhs_old = lhs; + lhs.makeDifference(rhs); + ASSERT_EQ(lhs, lhs_old); + + lhs = {}; + rhs = {}; + lhs.grant(AccessType::ALL); + rhs.grant(AccessType::ALL); + rhs.revoke(AccessType::SELECT, "system"); + lhs.makeDifference(rhs); + ASSERT_EQ(lhs.toString(), "GRANT SELECT ON system.*"); +} + TEST(AccessRights, Contains) { AccessRights lhs, rhs; diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 083b3a8f462..e46c301f9ea 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -423,6 +423,14 @@ QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalo else storage = DatabaseCatalog::instance().tryGetTable(storage_id, context); + if (!storage && storage_id.hasUUID()) + { + // If `storage_id` has UUID, it is possible that the UUID is removed from `DatabaseCatalog` after `context->resolveStorageID(storage_id)` + // We try to get the table with the database name and the table name. + auto database = DatabaseCatalog::instance().tryGetDatabase(storage_id.getDatabaseName()); + if (database) + storage = database->tryGetTable(table_name, context); + } if (!storage) return {}; diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index f353062f628..a60ac0c636f 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -570,7 +570,7 @@ void BackupCoordinationRemote::prepareReplicatedAccess() const if (replicated_access) return; - std::vector file_path_for_access_entities; + std::vector file_path_for_access_entities; auto holder = with_retries.createRetriesControlHolder("prepareReplicatedAccess"); holder.retries_ctl.retryLoop( [&, &zk = holder.faulty_zookeeper]() diff --git a/src/Backups/BackupCoordinationReplicatedAccess.cpp b/src/Backups/BackupCoordinationReplicatedAccess.cpp index 5a7d049920c..075ccadcc5d 100644 --- a/src/Backups/BackupCoordinationReplicatedAccess.cpp +++ b/src/Backups/BackupCoordinationReplicatedAccess.cpp @@ -1,5 +1,9 @@ #include +#include + +namespace fs = std::filesystem; + namespace DB { @@ -7,7 +11,7 @@ namespace DB BackupCoordinationReplicatedAccess::BackupCoordinationReplicatedAccess() = default; BackupCoordinationReplicatedAccess::~BackupCoordinationReplicatedAccess() = default; -void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntitry && file_path_for_access_entity) +void BackupCoordinationReplicatedAccess::addFilePath(FilePathForAccessEntity && file_path_for_access_entity) { const auto & access_zk_path = file_path_for_access_entity.access_zk_path; const auto & access_entity_type = file_path_for_access_entity.access_entity_type; @@ -28,10 +32,19 @@ Strings BackupCoordinationReplicatedAccess::getFilePaths(const String & access_z return {}; const auto & file_paths = it->second; - if (file_paths.host_to_store_access != host_id) + if ((file_paths.host_to_store_access != host_id) || file_paths.file_paths.empty()) return {}; - Strings res{file_paths.file_paths.begin(), file_paths.file_paths.end()}; + /// Use the same filename for all the paths in backup. + /// Those filenames have format "access-.txt", where UUID is random. + /// It's not really necessary, however it looks better if those files have the same filename + /// for a backup of ReplicatedAccessStorage on different hosts. + Strings res; + res.reserve(file_paths.file_paths.size()); + String filename = fs::path{*file_paths.file_paths.begin()}.filename(); + for (const auto & file_path : file_paths.file_paths) + res.emplace_back(fs::path{file_path}.replace_filename(filename)); + return res; } diff --git a/src/Backups/BackupCoordinationReplicatedAccess.h b/src/Backups/BackupCoordinationReplicatedAccess.h index 23672c3799e..34a7c915588 100644 --- a/src/Backups/BackupCoordinationReplicatedAccess.h +++ b/src/Backups/BackupCoordinationReplicatedAccess.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB @@ -28,7 +28,7 @@ public: BackupCoordinationReplicatedAccess(); ~BackupCoordinationReplicatedAccess(); - struct FilePathForAccessEntitry + struct FilePathForAccessEntity { String access_zk_path; AccessEntityType access_entity_type; @@ -37,7 +37,7 @@ public: }; /// Adds a path to access*.txt file keeping access entities of a ReplicatedAccessStorage. - void addFilePath(FilePathForAccessEntitry && file_path_for_access_entity); + void addFilePath(FilePathForAccessEntity && file_path_for_access_entity); /// Returns all paths added by addFilePath() if `host_id` is a host chosen to store access. Strings getFilePaths(const String & access_zk_path, AccessEntityType access_entity_type, const String & host_id) const; @@ -47,7 +47,7 @@ private: struct FilePathsAndHost { - std::unordered_set file_paths; + std::set file_paths; String host_to_store_access; }; diff --git a/src/Backups/BackupCoordinationStage.h b/src/Backups/BackupCoordinationStage.h index 0a414a59a77..9abdc019784 100644 --- a/src/Backups/BackupCoordinationStage.h +++ b/src/Backups/BackupCoordinationStage.h @@ -32,6 +32,9 @@ namespace BackupCoordinationStage /// Finding databases and tables in the backup which we're going to restore. constexpr const char * FINDING_TABLES_IN_BACKUP = "finding tables in backup"; + /// Loading system access tables and then checking if the current user has enough access to restore. + constexpr const char * CHECKING_ACCESS_RIGHTS = "checking access rights"; + /// Creating databases or finding them and checking their definitions. constexpr const char * CREATING_DATABASES = "creating databases"; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 2024d7772f9..752691dd9ce 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -910,11 +911,20 @@ void BackupEntriesCollector::runPostTasks() LOG_TRACE(log, "All post tasks successfully executed"); } -size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type) +std::unordered_map BackupEntriesCollector::getAllAccessEntities() { std::lock_guard lock(mutex); - access_counters.resize(static_cast(AccessEntityType::MAX)); - return access_counters[static_cast(type)]++; + if (!all_access_entities) + { + all_access_entities.emplace(); + auto entities_with_ids = context->getAccessControl().readAllWithIDs(); + for (const auto & [id, entity] : entities_with_ids) + { + if (entity->isBackupAllowed()) + all_access_entities->emplace(id, entity); + } + } + return *all_access_entities; } } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index fda5774105e..ae076a84c8b 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -21,7 +21,8 @@ class IBackupCoordination; class IDatabase; using DatabasePtr = std::shared_ptr; struct StorageID; -enum class AccessEntityType : uint8_t; +struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; class QueryStatus; using QueryStatusPtr = std::shared_ptr; @@ -49,6 +50,9 @@ public: ContextPtr getContext() const { return context; } const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return global_zookeeper_retries_info; } + /// Returns all access entities which can be put into a backup. + std::unordered_map getAllAccessEntities(); + /// Adds a backup entry which will be later returned by run(). /// These function can be called by implementations of IStorage::backupData() in inherited storage classes. void addBackupEntry(const String & file_name, BackupEntryPtr backup_entry); @@ -61,9 +65,6 @@ public: /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. void addPostTask(std::function task); - /// Returns an incremental counter used to backup access control. - size_t getAccessCounter(AccessEntityType type); - private: void calculateRootPathInBackup(); @@ -177,9 +178,10 @@ private: std::vector> previous_databases_metadata; std::vector> previous_tables_metadata; + std::optional> all_access_entities; + BackupEntries backup_entries; std::queue> post_tasks; - std::vector access_counters; ThreadPool & threadpool; std::mutex mutex; diff --git a/src/Backups/BackupSettings.cpp b/src/Backups/BackupSettings.cpp index e982a806b7c..9b8117c6587 100644 --- a/src/Backups/BackupSettings.cpp +++ b/src/Backups/BackupSettings.cpp @@ -37,6 +37,7 @@ namespace ErrorCodes M(Bool, check_parts) \ M(Bool, check_projection_parts) \ M(Bool, allow_backup_broken_projections) \ + M(Bool, write_access_entities_dependents) \ M(Bool, internal) \ M(String, host_id) \ M(OptionalUUID, backup_uuid) diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h index 0abeb897db4..8c2ea21df01 100644 --- a/src/Backups/BackupSettings.h +++ b/src/Backups/BackupSettings.h @@ -77,6 +77,11 @@ struct BackupSettings /// Allow to create backup with broken projections. bool allow_backup_broken_projections = false; + /// Whether dependents of access entities should be written along with the access entities. + /// For example, if a role is granted to a user and we're making a backup of system.roles (but not system.users) + /// this is whether the backup will contain information to grant the role to the corresponding user again. + bool write_access_entities_dependents = true; + /// Internal, should not be specified by user. /// Whether this backup is a part of a distributed backup created by BACKUP ON CLUSTER. bool internal = false; diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp index 8e60e8d129e..a8f6607e413 100644 --- a/src/Backups/RestoreSettings.cpp +++ b/src/Backups/RestoreSettings.cpp @@ -160,7 +160,8 @@ namespace M(UInt64, replica_num_in_backup) \ M(Bool, allow_non_empty_tables) \ M(RestoreAccessCreationMode, create_access) \ - M(Bool, allow_unresolved_access_dependencies) \ + M(Bool, skip_unresolved_access_dependencies) \ + M(Bool, update_access_entities_dependents) \ M(RestoreUDFCreationMode, create_function) \ M(Bool, allow_s3_native_copy) \ M(Bool, use_same_s3_credentials_for_base_backup) \ @@ -187,7 +188,12 @@ RestoreSettings RestoreSettings::fromRestoreQuery(const ASTBackupQuery & query) else LIST_OF_RESTORE_SETTINGS(GET_SETTINGS_FROM_RESTORE_QUERY_HELPER) - throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Unknown setting {}", setting.name); + + /// `allow_unresolved_access_dependencies` is an obsolete name. + if (setting.name == "allow_unresolved_access_dependencies") + res.skip_unresolved_access_dependencies = SettingFieldBool{setting.value}.value; + else + throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Unknown setting {}", setting.name); } } diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h index fe07a0a7208..dd65ce3ea9c 100644 --- a/src/Backups/RestoreSettings.h +++ b/src/Backups/RestoreSettings.h @@ -100,9 +100,25 @@ struct RestoreSettings /// How the RESTORE command will handle if an user (or role or profile) which it's going to restore already exists. RestoreAccessCreationMode create_access = RestoreAccessCreationMode::kCreateIfNotExists; - /// Skip dependencies of access entities which can't be resolved. - /// For example, if an user has a profile assigned and that profile is not in the backup and doesn't exist locally. - bool allow_unresolved_access_dependencies = false; + /// Ignore dependencies or dependents (with update_access_entities_dependents=true) of access entities in the case if they can't be resolved. + /// For example: if a backup contains a profile assigned to a user: `CREATE PROFILE p1; CREATE USER u1 SETTINGS PROFILE p1` + /// and now we're restoring only user `u1` and profile `p1` doesn't exists, then + /// this flag is whether RESTORE should continue with restoring user `u1` without assigning profile `p1`. + /// Another example: if a backup contains a role granted to a user: `CREATE USER u2; CREATE ROLE r2; GRANT r2 TO u2` + /// and now we're restoring only user `u2` and role `r2` doesn't exist, then + /// this flag is whether RESTORE should continue with restoring user `u2` without that grant. + /// If this flag is false then RESTORE will throw an exception in that case. + bool skip_unresolved_access_dependencies = false; + + /// Try to update dependents of restored access entities. + /// For example: if a backup contains a profile assigned to a user: `CREATE PROFILE p1; CREATE USER u1 SETTINGS PROFILE p1` + /// and now we're restoring only profile `p1` and user `u1` already exists, then + /// this flag is whether restored profile `p1` should be assigned to user `u1` again. + /// Another example, if a backup contains a role granted to a user: `CREATE USER u2; CREATE ROLE r2; GRANT r2 TO u2` + /// and now we're restoring only role `r2` and user `u2` already exists, then + /// this flag is whether restored role `r2` should be granted to user `u2` again. + /// If this flag is false then RESTORE won't update existing access entities. + bool update_access_entities_dependents = true; /// How the RESTORE command will handle if a user-defined function which it's going to restore already exists. RestoreUDFCreationMode create_function = RestoreUDFCreationMode::kCreateIfNotExists; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 021d4208fb9..eb4ba9424ff 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -141,6 +141,8 @@ void RestorerFromBackup::run(Mode mode) waitFutures(); /// Check access rights. + setStage(Stage::CHECKING_ACCESS_RIGHTS); + loadSystemAccessTables(); checkAccessForObjectsFoundInBackup(); if (mode == Mode::CHECK_ACCESS_ONLY) @@ -487,25 +489,6 @@ void RestorerFromBackup::findTableInBackupImpl(const QualifiedTableName & table_ res_table_info.partitions.emplace(); insertAtEnd(*res_table_info.partitions, *partitions); } - - /// Special handling for ACL-related system tables. - if (!restore_settings.structure_only && isSystemAccessTableName(table_name)) - { - if (!access_restorer) - access_restorer = std::make_unique(backup, restore_settings); - - try - { - /// addDataPath() will parse access*.txt files and extract access entities from them. - /// We need to do that early because we need those access entities to check access. - access_restorer->addDataPath(data_path_in_backup); - } - catch (Exception & e) - { - e.addMessage("While parsing data of {} from backup", tableNameWithTypeToString(table_name.database, table_name.table, false)); - throw; - } - } } void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names) @@ -629,6 +612,27 @@ size_t RestorerFromBackup::getNumTables() const return table_infos.size(); } +void RestorerFromBackup::loadSystemAccessTables() +{ + if (restore_settings.structure_only) + return; + + /// Special handling for ACL-related system tables. + std::lock_guard lock{mutex}; + for (const auto & [table_name, table_info] : table_infos) + { + if (isSystemAccessTableName(table_name)) + { + if (!access_restorer) + access_restorer = std::make_unique(backup, restore_settings); + access_restorer->addDataPath(table_info.data_path_in_backup); + } + } + + if (access_restorer) + access_restorer->loadFromBackup(); +} + void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const { AccessRightsElements required_access; @@ -713,6 +717,15 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const context->checkAccess(required_access); } +AccessEntitiesToRestore RestorerFromBackup::getAccessEntitiesToRestore(const String & data_path_in_backup) const +{ + std::lock_guard lock{mutex}; + if (!access_restorer) + return {}; + access_restorer->generateRandomIDsAndResolveDependencies(context->getAccessControl()); + return access_restorer->getEntitiesToRestore(data_path_in_backup); +} + void RestorerFromBackup::createDatabases() { Strings database_names; @@ -1071,19 +1084,6 @@ void RestorerFromBackup::runDataRestoreTasks() } } -std::vector> RestorerFromBackup::getAccessEntitiesToRestore() -{ - std::lock_guard lock{mutex}; - - if (!access_restorer || access_restored) - return {}; - - /// getAccessEntitiesToRestore() will return entities only when called first time (we don't want to restore the same entities again). - access_restored = true; - - return access_restorer->getAccessEntities(context->getAccessControl()); -} - void RestorerFromBackup::throwTableIsNotEmpty(const StorageID & storage_id) { throw Exception( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 6dbca5bced5..e0130ccfcb4 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -20,8 +20,7 @@ struct StorageID; class IDatabase; using DatabasePtr = std::shared_ptr; class AccessRestorerFromBackup; -struct IAccessEntity; -using AccessEntityPtr = std::shared_ptr; +struct AccessEntitiesToRestore; class QueryStatus; using QueryStatusPtr = std::shared_ptr; @@ -68,7 +67,7 @@ public: void addDataRestoreTasks(DataRestoreTasks && new_tasks); /// Returns the list of access entities to restore. - std::vector> getAccessEntitiesToRestore(); + AccessEntitiesToRestore getAccessEntitiesToRestore(const String & data_path_in_backup) const; /// Throws an exception that a specified table is already non-empty. [[noreturn]] static void throwTableIsNotEmpty(const StorageID & storage_id); @@ -101,6 +100,7 @@ private: size_t getNumDatabases() const; size_t getNumTables() const; + void loadSystemAccessTables(); void checkAccessForObjectsFoundInBackup() const; void createDatabases(); diff --git a/src/Backups/tests/gtest_backup_entries.cpp b/src/Backups/tests/gtest_backup_entries.cpp index 2d5b993b95d..a18e1706c59 100644 --- a/src/Backups/tests/gtest_backup_entries.cpp +++ b/src/Backups/tests/gtest_backup_entries.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -217,8 +218,9 @@ TEST_F(BackupEntriesTest, PartialChecksumBeforeFullChecksum) TEST_F(BackupEntriesTest, BackupEntryFromSmallFile) { + auto read_settings = getContext().context->getReadSettings(); writeFile(local_disk, "a.txt"); - auto entry = std::make_shared(local_disk, "a.txt", ReadSettings{}); + auto entry = std::make_shared(local_disk, "a.txt", read_settings); local_disk->removeFile("a.txt"); @@ -234,12 +236,13 @@ TEST_F(BackupEntriesTest, BackupEntryFromSmallFile) TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk) { + auto read_settings = getContext().context->getReadSettings(); { writeFile(encrypted_disk, "a.txt"); std::pair test_cases[] = {{std::make_shared(encrypted_disk, "a.txt"), false}, {std::make_shared(encrypted_disk, "a.txt"), true}, - {std::make_shared(encrypted_disk, "a.txt", ReadSettings{}), true}}; + {std::make_shared(encrypted_disk, "a.txt", read_settings), true}}; for (const auto & [entry, partial_checksum_allowed] : test_cases) { EXPECT_EQ(entry->getSize(), 9); @@ -258,7 +261,7 @@ TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk) BackupEntryPtr entries[] = {std::make_shared(encrypted_disk, "empty.txt"), std::make_shared(encrypted_disk, "empty.txt"), - std::make_shared(encrypted_disk, "empty.txt", ReadSettings{})}; + std::make_shared(encrypted_disk, "empty.txt", read_settings)}; for (const auto & entry : entries) { EXPECT_EQ(entry->getSize(), 0); @@ -283,12 +286,13 @@ TEST_F(BackupEntriesTest, DecryptedEntriesFromEncryptedDisk) TEST_F(BackupEntriesTest, EncryptedEntriesFromEncryptedDisk) { + auto read_settings = getContext().context->getReadSettings(); { writeFile(encrypted_disk, "a.txt"); BackupEntryPtr entries[] = {std::make_shared(encrypted_disk, "a.txt", /* copy_encrypted= */ true), std::make_shared(encrypted_disk, "a.txt", /* copy_encrypted= */ true), - std::make_shared(encrypted_disk, "a.txt", ReadSettings{}, /* copy_encrypted= */ true)}; + std::make_shared(encrypted_disk, "a.txt", read_settings, /* copy_encrypted= */ true)}; auto encrypted_checksum = getChecksum(entries[0]); EXPECT_NE(encrypted_checksum, NO_CHECKSUM); @@ -322,7 +326,7 @@ TEST_F(BackupEntriesTest, EncryptedEntriesFromEncryptedDisk) BackupEntryPtr entries[] = {std::make_shared(encrypted_disk, "empty.txt", /* copy_encrypted= */ true), std::make_shared(encrypted_disk, "empty.txt", /* copy_encrypted= */ true), - std::make_shared(encrypted_disk, "empty.txt", ReadSettings{}, /* copy_encrypted= */ true)}; + std::make_shared(encrypted_disk, "empty.txt", read_settings, /* copy_encrypted= */ true)}; for (const auto & entry : entries) { EXPECT_EQ(entry->getSize(), 0); diff --git a/src/BridgeHelper/XDBCBridgeHelper.h b/src/BridgeHelper/XDBCBridgeHelper.h index 0630a0f24e3..b75ccbeedc9 100644 --- a/src/BridgeHelper/XDBCBridgeHelper.h +++ b/src/BridgeHelper/XDBCBridgeHelper.h @@ -243,7 +243,7 @@ protected: throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Failed to parse quoting style from '{}' for service {}", character, BridgeHelperMixin::serviceAlias()); else if (character.empty()) - quote_style = IdentifierQuotingStyle::None; + quote_style = IdentifierQuotingStyle::Backticks; else if (character[0] == '`') quote_style = IdentifierQuotingStyle::Backticks; else if (character[0] == '"') diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index a61ea4c8156..b2dd41715db 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2825,11 +2825,12 @@ void ClientBase::runLibFuzzer() {} void ClientBase::clearTerminal() { - /// Clear from cursor until end of screen. + /// Move to the beginning of the line + /// and clear until end of screen. /// It is needed if garbage is left in terminal. /// Show cursor. It can be left hidden by invocation of previous programs. /// A test for this feature: perl -e 'print "x"x100000'; echo -ne '\033[0;0H\033[?25l'; clickhouse-client - output_stream << "\033[0J" "\033[?25h"; + output_stream << "\r" "\033[0J" "\033[?25h"; } void ClientBase::showClientVersion() diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index affd620f83a..657a88c00c8 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -85,7 +84,6 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti add_column("name", "columns", true, suggestion_limit); } - /// FIXME: This query does not work with the new analyzer because of bug https://github.com/ClickHouse/ClickHouse/issues/50669 query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)"; return query; } diff --git a/src/Common/ActionBlocker.h b/src/Common/ActionBlocker.h index 3336a311af3..b0a248e24f7 100644 --- a/src/Common/ActionBlocker.h +++ b/src/Common/ActionBlocker.h @@ -17,7 +17,7 @@ public: bool isCancelled() const { return *counter > 0; } - /// Temporarily blocks corresponding actions (while the returned object is alive) + /// Temporary blocks corresponding actions (while the returned object is alive) friend class ActionLock; [[nodiscard]] ActionLock cancel() { return ActionLock(*this); } diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h index 0a11e52be2c..2d4ad36f7f2 100644 --- a/src/Common/CounterInFile.h +++ b/src/Common/CounterInFile.h @@ -112,12 +112,12 @@ public: } catch (...) { - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); throw; } - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); return res; } @@ -182,12 +182,12 @@ public: } catch (...) { - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); throw; } - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); } diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index ef7c6e143a0..46cadfd2ac1 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -95,7 +95,7 @@ Epoll::~Epoll() { if (epoll_fd != -1) { - int err = close(epoll_fd); + [[maybe_unused]] int err = close(epoll_fd); chassert(!err || errno == EINTR); } } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 09a5375191b..9f07c3ed5d5 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -452,7 +452,7 @@ M(553, LZMA_STREAM_ENCODER_FAILED) \ M(554, LZMA_STREAM_DECODER_FAILED) \ M(555, ROCKSDB_ERROR) \ - M(556, SYNC_MYSQL_USER_ACCESS_ERROR)\ + M(556, SYNC_MYSQL_USER_ACCESS_ERROR) \ M(557, UNKNOWN_UNION) \ M(558, EXPECTED_ALL_OR_DISTINCT) \ M(559, INVALID_GRPC_QUERY_INFO) \ @@ -578,7 +578,7 @@ M(697, CANNOT_RESTORE_TO_NONENCRYPTED_DISK) \ M(698, INVALID_REDIS_STORAGE_TYPE) \ M(699, INVALID_REDIS_TABLE_STRUCTURE) \ - M(700, USER_SESSION_LIMIT_EXCEEDED) \ + M(700, USER_SESSION_LIMIT_EXCEEDED) \ M(701, CLUSTER_DOESNT_EXIST) \ M(702, CLIENT_INFO_DOES_NOT_MATCH) \ M(703, INVALID_IDENTIFIER) \ @@ -610,15 +610,16 @@ M(729, ILLEGAL_TIME_SERIES_TAGS) \ M(730, REFRESH_FAILED) \ M(731, QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE) \ - \ + M(733, TABLE_IS_BEING_RESTARTED) \ +\ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ - \ +\ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ M(1001, STD_EXCEPTION) \ M(1002, UNKNOWN_EXCEPTION) \ -/* See END */ + /* See END */ #ifdef APPLY_FOR_EXTERNAL_ERROR_CODES #define APPLY_FOR_ERROR_CODES(M) APPLY_FOR_BUILTIN_ERROR_CODES(M) APPLY_FOR_EXTERNAL_ERROR_CODES(M) diff --git a/src/Common/EventFD.cpp b/src/Common/EventFD.cpp index 9ec7f128420..9443bad93d2 100644 --- a/src/Common/EventFD.cpp +++ b/src/Common/EventFD.cpp @@ -57,7 +57,7 @@ EventFD::~EventFD() { if (fd != -1) { - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); } } diff --git a/src/Common/EventFD.h b/src/Common/EventFD.h index 95ecdc20dd6..ccf03165119 100644 --- a/src/Common/EventFD.h +++ b/src/Common/EventFD.h @@ -2,7 +2,6 @@ #if defined(OS_LINUX) -#include #include diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index 5ecbe44530b..01fa91dd4ef 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -176,7 +176,7 @@ void FileChecker::load() if (!fileReallyExists(files_info_path)) return; - std::unique_ptr in = disk ? disk->readFile(files_info_path) : std::make_unique(files_info_path); + std::unique_ptr in = disk ? disk->readFile(files_info_path, getReadSettings()) : std::make_unique(files_info_path); WriteBufferFromOwnString out; /// The JSON library does not support whitespace. We delete them. Inefficient. diff --git a/src/Common/NetlinkMetricsProvider.cpp b/src/Common/NetlinkMetricsProvider.cpp index b12d81df45c..f30186e715d 100644 --- a/src/Common/NetlinkMetricsProvider.cpp +++ b/src/Common/NetlinkMetricsProvider.cpp @@ -283,7 +283,7 @@ NetlinkMetricsProvider::NetlinkMetricsProvider() { if (netlink_socket_fd >= 0) { - int err = close(netlink_socket_fd); + [[maybe_unused]] int err = close(netlink_socket_fd); chassert(!err || errno == EINTR); } throw; @@ -320,7 +320,7 @@ NetlinkMetricsProvider::~NetlinkMetricsProvider() { if (netlink_socket_fd >= 0) { - int err = close(netlink_socket_fd); + [[maybe_unused]] int err = close(netlink_socket_fd); chassert(!err || errno == EINTR); } } diff --git a/src/Common/ProcfsMetricsProvider.cpp b/src/Common/ProcfsMetricsProvider.cpp index 194053cd271..c28a68d9032 100644 --- a/src/Common/ProcfsMetricsProvider.cpp +++ b/src/Common/ProcfsMetricsProvider.cpp @@ -98,7 +98,7 @@ ProcfsMetricsProvider::ProcfsMetricsProvider(pid_t /*tid*/) thread_stat_fd = ::open(thread_stat, O_RDONLY | O_CLOEXEC); if (-1 == thread_stat_fd) { - int err = ::close(thread_schedstat_fd); + [[maybe_unused]] int err = ::close(thread_schedstat_fd); chassert(!err || errno == EINTR); throwWithFailedToOpenFile(thread_stat); } diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index 0bbb7ff411d..3e7c5d4788b 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -100,7 +100,7 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_) } catch (...) { - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); throw; } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 7274cdfb9f1..d6f1716c781 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -578,7 +578,7 @@ public: virtual String getConnectedHostPort() const = 0; /// Get the xid of current connection. - virtual int32_t getConnectionXid() const = 0; + virtual int64_t getConnectionXid() const = 0; /// Useful to check owner of ephemeral node. virtual int64_t getSessionID() const = 0; diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index c32f0064dec..4cf08c9c8f6 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -41,7 +41,7 @@ public: bool isExpired() const override { return expired; } std::optional getConnectedNodeIdx() const override { return 0; } String getConnectedHostPort() const override { return "TestKeeper:0000"; } - int32_t getConnectionXid() const override { return 0; } + int64_t getConnectionXid() const override { return 0; } int64_t getSessionID() const override { return 0; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index aebf7c17e3c..fb6d0a27d5c 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -1565,7 +1565,7 @@ String ZooKeeper::getConnectedHostPort() const return impl->getConnectedHostPort(); } -int32_t ZooKeeper::getConnectionXid() const +int64_t ZooKeeper::getConnectionXid() const { return impl->getConnectionXid(); } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 15359a6381a..a8711f7d31a 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -627,7 +627,7 @@ public: std::optional getConnectedHostIdx() const; String getConnectedHostPort() const; - int32_t getConnectionXid() const; + int64_t getConnectionXid() const; String getConnectedHostAvailabilityZone() const; diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index 18dff779a70..6d18b2cc73d 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -101,6 +101,8 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig session_timeout_ms = config.getInt(session_timeout_key); } + use_xid_64 = config.getBool(std::string{config_name} + ".use_xid_64", false); + Poco::Util::AbstractConfiguration::Keys keys; std::string raft_configuration_key = std::string{config_name} + ".raft_configuration"; config.keys(raft_configuration_key, keys); @@ -241,6 +243,10 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio { use_compression = config.getBool(config_name + "." + key); } + else if (key == "use_xid_64") + { + use_xid_64 = config.getBool(config_name + "." + key); + } else if (key == "availability_zone_autodetect") { availability_zone_autodetect = config.getBool(config_name + "." + key); diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.h b/src/Common/ZooKeeper/ZooKeeperArgs.h index 945b77bf9c1..3754c2f7aac 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.h +++ b/src/Common/ZooKeeper/ZooKeeperArgs.h @@ -49,6 +49,7 @@ struct ZooKeeperArgs UInt64 send_sleep_ms = 0; UInt64 recv_sleep_ms = 0; bool use_compression = false; + bool use_xid_64 = false; bool prefer_local_availability_zone = false; bool availability_zone_autodetect = false; diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index cd2cefb37bd..1dfa3040827 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -16,14 +16,23 @@ namespace Coordination using namespace DB; -void ZooKeeperResponse::write(WriteBuffer & out) const +void ZooKeeperResponse::write(WriteBuffer & out, bool use_xid_64) const { - auto response_size = Coordination::size(xid) + Coordination::size(zxid) + Coordination::size(error); + size_t response_size = 0; + if (use_xid_64) + response_size += sizeof(int64_t); + else + response_size += sizeof(int32_t); + + response_size += Coordination::size(zxid) + Coordination::size(error); if (error == Error::ZOK) response_size += sizeImpl(); Coordination::write(static_cast(response_size), out); - Coordination::write(xid, out); + if (use_xid_64) + Coordination::write(xid, out); + else + Coordination::write(static_cast(xid), out); Coordination::write(zxid, out); Coordination::write(error, out); if (error == Error::ZOK) @@ -41,12 +50,21 @@ std::string ZooKeeperRequest::toString(bool short_format) const toStringImpl(short_format)); } -void ZooKeeperRequest::write(WriteBuffer & out) const +void ZooKeeperRequest::write(WriteBuffer & out, bool use_xid_64) const { - auto request_size = Coordination::size(xid) + Coordination::size(getOpNum()) + sizeImpl(); + size_t request_size = 0; + if (use_xid_64) + request_size += sizeof(int64_t); + else + request_size += sizeof(int32_t); + + request_size += Coordination::size(getOpNum()) + sizeImpl(); Coordination::write(static_cast(request_size), out); - Coordination::write(xid, out); + if (use_xid_64) + Coordination::write(static_cast(xid), out); + else + Coordination::write(static_cast(xid), out); Coordination::write(getOpNum(), out); writeImpl(out); } @@ -150,10 +168,10 @@ size_t ZooKeeperWatchResponse::sizeImpl() const return Coordination::size(type) + Coordination::size(state) + Coordination::size(path); } -void ZooKeeperWatchResponse::write(WriteBuffer & out) const +void ZooKeeperWatchResponse::write(WriteBuffer & out, bool use_xid_64) const { if (error == Error::ZOK) - ZooKeeperResponse::write(out); + ZooKeeperResponse::write(out, use_xid_64); /// skip bad responses for watches } @@ -732,15 +750,13 @@ void ZooKeeperMultiRequest::writeImpl(WriteBuffer & out) const size_t ZooKeeperMultiRequest::sizeImpl() const { size_t total_size = 0; - for (const auto & request : requests) + for (const auto & zk_request : requests) { - const auto & zk_request = dynamic_cast(*request); - bool done = false; int32_t error = -1; total_size - += Coordination::size(zk_request.getOpNum()) + Coordination::size(done) + Coordination::size(error) + zk_request.sizeImpl(); + += Coordination::size(zk_request->getOpNum()) + Coordination::size(done) + Coordination::size(error) + zk_request->sizeImpl(); } OpNum op_num = OpNum::Error; diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index cd1b2ff250b..123ddf956fc 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -28,7 +28,7 @@ struct ZooKeeperResponse : virtual Response virtual void readImpl(ReadBuffer &) = 0; virtual void writeImpl(WriteBuffer &) const = 0; virtual size_t sizeImpl() const = 0; - virtual void write(WriteBuffer & out) const; + virtual void write(WriteBuffer & out, bool use_xid_64) const; virtual OpNum getOpNum() const = 0; virtual void fillLogElements(LogElements & elems, size_t idx) const; virtual int32_t tryGetOpNum() const { return static_cast(getOpNum()); } @@ -56,7 +56,7 @@ struct ZooKeeperRequest : virtual Request virtual OpNum getOpNum() const = 0; /// Writes length, xid, op_num, then the rest. - void write(WriteBuffer & out) const; + void write(WriteBuffer & out, bool use_xid_64) const; std::string toString(bool short_format = false) const; virtual void writeImpl(WriteBuffer &) const = 0; @@ -155,7 +155,7 @@ struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse void writeImpl(WriteBuffer & out) const override; size_t sizeImpl() const override; - void write(WriteBuffer & out) const override; + void write(WriteBuffer & out, bool use_xid_64) const override; OpNum getOpNum() const override { diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 9d8e2d4f857..d1213d4d72f 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -8,12 +8,13 @@ namespace Coordination { -using XID = int32_t; +using XID = int64_t; static constexpr XID WATCH_XID = -1; static constexpr XID PING_XID = -2; static constexpr XID AUTH_XID = -4; -static constexpr XID CLOSE_XID = 0x7FFFFFFF; +static constexpr XID CLOSE_XID = std::numeric_limits::max(); +static constexpr XID CLOSE_XID_64 = std::numeric_limits::max(); enum class OpNum : int32_t { @@ -49,6 +50,7 @@ OpNum getOpNum(int32_t raw_op_num); static constexpr int32_t ZOOKEEPER_PROTOCOL_VERSION = 0; static constexpr int32_t ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION = 10; +static constexpr int32_t ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64 = 11; static constexpr int32_t KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT = 42; static constexpr int32_t CLIENT_HANDSHAKE_LENGTH = 44; static constexpr int32_t CLIENT_HANDSHAKE_LENGTH_WITH_READONLY = 45; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index a6dd9738e17..510ebd4f82d 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -378,6 +378,11 @@ ZooKeeper::ZooKeeper( try { use_compression = args.use_compression; + if (args.use_xid_64) + { + use_xid_64 = true; + close_xid = CLOSE_XID_64; + } connect(nodes, args.connection_timeout_ms * 1000); } catch (...) @@ -590,10 +595,19 @@ void ZooKeeper::sendHandshake() bool read_only = true; write(handshake_length); - if (use_compression) + if (use_xid_64) + { + write(ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64); + write(use_compression); + } + else if (use_compression) + { write(ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION); + } else + { write(ZOOKEEPER_PROTOCOL_VERSION); + } write(last_zxid_seen); write(timeout); write(previous_session_id); @@ -624,10 +638,15 @@ void ZooKeeper::receiveHandshake() "Keeper server rejected the connection during the handshake. " "Possibly it's overloaded, doesn't see leader or stale"); - if (use_compression) + if (use_xid_64) + { + if (protocol_version_read < ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64) + throw Exception(Error::ZMARSHALLINGERROR, "Unexpected protocol version with 64bit XID: {}", protocol_version_read); + } + else if (use_compression) { if (protocol_version_read != ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION) - throw Exception(Error::ZMARSHALLINGERROR,"Unexpected protocol version with compression: {}", protocol_version_read); + throw Exception(Error::ZMARSHALLINGERROR, "Unexpected protocol version with compression: {}", protocol_version_read); } else if (protocol_version_read != ZOOKEEPER_PROTOCOL_VERSION) throw Exception(Error::ZMARSHALLINGERROR, "Unexpected protocol version: {}", protocol_version_read); @@ -650,7 +669,7 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) request.scheme = scheme; request.data = data; request.xid = AUTH_XID; - request.write(getWriteBuffer()); + request.write(getWriteBuffer(), use_xid_64); flushWriteBuffer(); int32_t length; @@ -660,7 +679,17 @@ void ZooKeeper::sendAuth(const String & scheme, const String & data) read(length); size_t count_before_event = in->count(); - read(read_xid); + if (use_xid_64) + { + read(read_xid); + } + else + { + int32_t xid_32{0}; + read(xid_32); + read_xid = xid_32; + } + read(zxid); read(err); @@ -710,7 +739,7 @@ void ZooKeeper::sendThread() /// After we popped element from the queue, we must register callbacks (even in the case when expired == true right now), /// because they must not be lost (callbacks must be called because the user will wait for them). - if (info.request->xid != CLOSE_XID) + if (info.request->xid != close_xid) { CurrentMetrics::add(CurrentMetrics::ZooKeeperRequest); std::lock_guard lock(operations_mutex); @@ -730,13 +759,13 @@ void ZooKeeper::sendThread() info.request->addRootPath(args.chroot); info.request->probably_sent = true; - info.request->write(getWriteBuffer()); + info.request->write(getWriteBuffer(), use_xid_64); flushWriteBuffer(); logOperationIfNeeded(info.request); /// We sent close request, exit - if (info.request->xid == CLOSE_XID) + if (info.request->xid == close_xid) break; } } @@ -747,7 +776,7 @@ void ZooKeeper::sendThread() ZooKeeperHeartbeatRequest request; request.xid = PING_XID; - request.write(getWriteBuffer()); + request.write(getWriteBuffer(), use_xid_64); flushWriteBuffer(); } @@ -833,7 +862,16 @@ void ZooKeeper::receiveEvent() read(length); size_t count_before_event = in->count(); - read(xid); + if (use_xid_64) + { + read(xid); + } + else + { + int32_t xid_32{0}; + read(xid_32); + xid = xid_32; + } read(zxid); read(err); @@ -1191,7 +1229,7 @@ void ZooKeeper::pushRequest(RequestInfo && info) if (!info.request->xid) { info.request->xid = next_xid.fetch_add(1); - if (info.request->xid == CLOSE_XID) + if (info.request->xid == close_xid) throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "xid equal to close_xid"); if (info.request->xid < 0) throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "XID overflow"); @@ -1538,7 +1576,7 @@ void ZooKeeper::multi( void ZooKeeper::close() { ZooKeeperCloseRequest request; - request.xid = CLOSE_XID; + request.xid = close_xid; RequestInfo request_info; request_info.request = std::make_shared(std::move(request)); @@ -1568,7 +1606,7 @@ String ZooKeeper::getConnectedHostPort() const return ""; } -int32_t ZooKeeper::getConnectionXid() const +int64_t ZooKeeper::getConnectionXid() const { return next_xid.load(); } diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 47d2ab8f401..2426731cdeb 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -119,7 +119,7 @@ public: std::optional getConnectedNodeIdx() const override; String getConnectedHostPort() const override; - int32_t getConnectionXid() const override; + int64_t getConnectionXid() const override; String tryGetAvailabilityZone() override; @@ -247,6 +247,9 @@ private: std::optional compressed_out; bool use_compression = false; + bool use_xid_64 = false; + + int64_t close_xid = CLOSE_XID; int64_t session_id = 0; diff --git a/src/Common/tests/gtest_action_blocker.cpp b/src/Common/tests/gtest_action_blocker.cpp new file mode 100644 index 00000000000..f077d5e73dd --- /dev/null +++ b/src/Common/tests/gtest_action_blocker.cpp @@ -0,0 +1,128 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace DB; + +TEST(ActionBlocker, TestDefaultConstructor) +{ + ActionBlocker blocker; + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_EQ(0, blocker.getCounter().load()); +} + +TEST(ActionBlocker, TestCancelForever) +{ + ActionBlocker blocker; + + blocker.cancelForever(); + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_EQ(1, blocker.getCounter().load()); +} + +TEST(ActionBlocker, TestCancel) +{ + ActionBlocker blocker; + + { + auto lock = blocker.cancel(); + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_EQ(1, blocker.getCounter().load()); + } + // automatically un-cancelled on `lock` destruction + EXPECT_FALSE(blocker.isCancelled()); +} + + + +TEST(ActionLock, TestDefaultConstructor) +{ + ActionLock locker; + EXPECT_TRUE(locker.expired()); +} + +TEST(ActionLock, TestConstructorWithActionBlocker) +{ + ActionBlocker blocker; + ActionLock lock(blocker); + + EXPECT_FALSE(lock.expired()); + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_EQ(1, blocker.getCounter().load()); +} + +TEST(ActionLock, TestMoveAssignmentToEmpty) +{ + ActionBlocker blocker; + + { + ActionLock lock; + lock = blocker.cancel(); + EXPECT_TRUE(blocker.isCancelled()); + } + // automatically un-cancelled on `lock` destruction + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_EQ(0, blocker.getCounter().load()); +} + +TEST(ActionLock, TestMoveAssignmentToNonEmpty) +{ + ActionBlocker blocker; + { + auto lock = blocker.cancel(); + EXPECT_TRUE(blocker.isCancelled()); + + // cause a move + lock = blocker.cancel(); + + // blocker should remain locked + EXPECT_TRUE(blocker.isCancelled()); + } + // automatically un-cancelled on `lock` destruction + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_EQ(0, blocker.getCounter().load()); +} + +TEST(ActionLock, TestMoveAssignmentToNonEmpty2) +{ + ActionBlocker blocker1; + ActionBlocker blocker2; + { + auto lock = blocker1.cancel(); + EXPECT_TRUE(blocker1.isCancelled()); + + // cause a move + lock = blocker2.cancel(); + + // blocker2 be remain locked, blocker1 - unlocked + EXPECT_TRUE(blocker2.isCancelled()); + + EXPECT_FALSE(blocker1.isCancelled()); + } + // automatically un-cancelled on `lock` destruction + EXPECT_FALSE(blocker1.isCancelled()); + EXPECT_FALSE(blocker2.isCancelled()); + EXPECT_EQ(0, blocker1.getCounter().load()); + EXPECT_EQ(0, blocker2.getCounter().load()); +} + +TEST(ActionLock, TestExpiration) +{ + ActionLock lock; + { + ActionBlocker blocker; + lock = blocker.cancel(); + EXPECT_FALSE(lock.expired()); + } + + EXPECT_TRUE(lock.expired()); +} diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 7ae8bb32f70..a2d09272a31 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -1,6 +1,12 @@ #pragma once #include +#include + +namespace DB::Setting +{ + extern const SettingsString local_filesystem_read_method; +} struct ContextHolder { @@ -13,6 +19,7 @@ struct ContextHolder { context->makeGlobalContext(); context->setPath("./"); + const_cast(context->getSettingsRef())[DB::Setting::local_filesystem_read_method] = "pread"; } ContextHolder(ContextHolder &&) = default; diff --git a/src/Common/waitForPid.cpp b/src/Common/waitForPid.cpp index 16c447245a2..c5940402d46 100644 --- a/src/Common/waitForPid.cpp +++ b/src/Common/waitForPid.cpp @@ -112,7 +112,7 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms) if (ready <= 0) return PollPidResult::FAILED; - int err = close(pid_fd); + [[maybe_unused]] int err = close(pid_fd); chassert(!err || errno == EINTR); return PollPidResult::RESTART; diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 9607c345a3b..89034951f94 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -217,7 +217,7 @@ public: std::move(file_buf), /* compressi)on level = */ 3, /* append_to_existing_file_ = */ mode == WriteMode::Append, - [latest_log_disk, path = current_file_description->path] { return latest_log_disk->readFile(path); }); + [latest_log_disk, path = current_file_description->path, read_settings = getReadSettings()] { return latest_log_disk->readFile(path, read_settings); }); prealloc_done = false; } @@ -601,7 +601,7 @@ public: explicit ChangelogReader(ChangelogFileDescriptionPtr changelog_description_) : changelog_description(changelog_description_) { compression_method = chooseCompressionMethod(changelog_description->path, ""); - auto read_buffer_from_file = changelog_description->disk->readFile(changelog_description->path); + auto read_buffer_from_file = changelog_description->disk->readFile(changelog_description->path, getReadSettings()); read_buf = wrapReadBufferWithCompressionMethod(std::move(read_buffer_from_file), compression_method); } @@ -728,7 +728,7 @@ void LogEntryStorage::prefetchCommitLogs() [&] { const auto & [changelog_description, position, count] = prefetch_file_info; - auto file = changelog_description->disk->readFile(changelog_description->path, ReadSettings()); + auto file = changelog_description->disk->readFile(changelog_description->path, getReadSettings()); file->seek(position, SEEK_SET); LOG_TRACE( log, "Prefetching {} log entries from path {}, from position {}", count, changelog_description->path, position); @@ -1266,7 +1266,7 @@ LogEntryPtr LogEntryStorage::getEntry(uint64_t index) const [&] { const auto & [changelog_description, position, size] = it->second; - auto file = changelog_description->disk->readFile(changelog_description->path, ReadSettings()); + auto file = changelog_description->disk->readFile(changelog_description->path, getReadSettings()); file->seek(position, SEEK_SET); LOG_TRACE( log, @@ -1392,7 +1392,7 @@ LogEntriesPtr LogEntryStorage::getLogEntriesBetween(uint64_t start, uint64_t end [&] { const auto & [file_description, start_position, count] = *read_info; - auto file = file_description->disk->readFile(file_description->path); + auto file = file_description->disk->readFile(file_description->path, getReadSettings()); file->seek(start_position, SEEK_SET); for (size_t i = 0; i < count; ++i) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index bfe50288a40..f54e5b8ca9d 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -597,20 +597,6 @@ void KeeperServer::shutdown() namespace { -// Serialize the request for the log entry -nuraft::ptr getZooKeeperLogEntry(const KeeperStorageBase::RequestForSession & request_for_session) -{ - DB::WriteBufferFromNuraftBuffer write_buf; - DB::writeIntBinary(request_for_session.session_id, write_buf); - request_for_session.request->write(write_buf); - DB::writeIntBinary(request_for_session.time, write_buf); - /// we fill with dummy values to eliminate unnecessary copy later on when we will write correct values - DB::writeIntBinary(static_cast(0), write_buf); /// zxid - DB::writeIntBinary(KeeperStorageBase::DigestVersion::NO_DIGEST, write_buf); /// digest version or NO_DIGEST flag - DB::writeIntBinary(static_cast(0), write_buf); /// digest value - /// if new fields are added, update KeeperStateMachine::ZooKeeperLogSerializationVersion along with parseRequest function and PreAppendLog callback handler - return write_buf.getBuffer(); -} } @@ -627,7 +613,7 @@ RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorageBase::Requests std::vector> entries; entries.reserve(requests_for_sessions.size()); for (const auto & request_for_session : requests_for_sessions) - entries.push_back(getZooKeeperLogEntry(request_for_session)); + entries.push_back(IKeeperStateMachine::getZooKeeperLogEntry(request_for_session)); std::lock_guard lock{server_write_mutex}; if (is_recovering) @@ -881,6 +867,9 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_ZXID_DIGEST) bytes_missing += sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + sizeof(request_for_session->digest->value); + if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_XID_64) + bytes_missing += sizeof(uint32_t); + if (bytes_missing != 0) { auto new_buffer = nuraft::buffer::alloc(entry_buf->size() + bytes_missing); @@ -889,8 +878,8 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ entry = nuraft::cs_new(entry->get_term(), entry_buf, entry->get_val_type()); } - size_t write_buffer_header_size - = sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + sizeof(request_for_session->digest->value); + size_t write_buffer_header_size = sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + + sizeof(request_for_session->digest->value) + sizeof(uint32_t); if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME) write_buffer_header_size += sizeof(request_for_session->time); diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 19335835041..231f50c7f81 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -729,7 +729,7 @@ nuraft::ptr KeeperSnapshotManager::deserializeSnapshotB { const auto & [snapshot_path, snapshot_disk, size] = *existing_snapshots.at(up_to_log_idx); WriteBufferFromNuraftBuffer writer; - auto reader = snapshot_disk->readFile(snapshot_path); + auto reader = snapshot_disk->readFile(snapshot_path, getReadSettings()); copyData(*reader, writer); return writer.getBuffer(); } diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 66ac2be810e..0480314647b 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -173,7 +173,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh LOG_INFO(log, "Will try to upload snapshot on {} to S3", snapshot_path); - auto snapshot_file = snapshot_disk->readFile(snapshot_path); + auto snapshot_file = snapshot_disk->readFile(snapshot_path, getReadSettings()); auto snapshot_name = fs::path(snapshot_path).filename().string(); auto lock_file = fmt::format(".{}_LOCK", snapshot_name); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index b3c5465fee1..16a76045f27 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -221,7 +221,47 @@ nuraft::ptr KeeperStateMachine::pre_commit(uint64_t log return result; } -std::shared_ptr IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version) +namespace +{ + +union XidHelper +{ + struct + { + uint32_t lower; + uint32_t upper; + } parts; + int64_t xid; +}; + +}; + +// Serialize the request for the log entry +nuraft::ptr IKeeperStateMachine::getZooKeeperLogEntry(const KeeperStorageBase::RequestForSession & request_for_session) +{ + DB::WriteBufferFromNuraftBuffer write_buf; + DB::writeIntBinary(request_for_session.session_id, write_buf); + + const auto & request = request_for_session.request; + size_t request_size = sizeof(uint32_t) + Coordination::size(request->getOpNum()) + request->sizeImpl(); + Coordination::write(static_cast(request_size), write_buf); + XidHelper xid_helper{.xid = request->xid}; + Coordination::write(xid_helper.parts.lower, write_buf); + Coordination::write(request->getOpNum(), write_buf); + request->writeImpl(write_buf); + + DB::writeIntBinary(request_for_session.time, write_buf); + /// we fill with dummy values to eliminate unnecessary copy later on when we will write correct values + DB::writeIntBinary(static_cast(0), write_buf); /// zxid + DB::writeIntBinary(KeeperStorageBase::DigestVersion::NO_DIGEST, write_buf); /// digest version or NO_DIGEST flag + DB::writeIntBinary(static_cast(0), write_buf); /// digest value + Coordination::write(xid_helper.parts.upper, write_buf); /// for 64bit XID MSB + /// if new fields are added, update KeeperStateMachine::ZooKeeperLogSerializationVersion along with parseRequest function and PreAppendLog callback handler + return write_buf.getBuffer(); +} + +std::shared_ptr +IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version) { ReadBufferFromNuraftBuffer buffer(data); auto request_for_session = std::make_shared(); @@ -230,14 +270,62 @@ std::shared_ptr IKeeperStateMachine::parse int32_t length; Coordination::read(length, buffer); - int32_t xid; - Coordination::read(xid, buffer); + /// because of backwards compatibility, only 32bit xid could be written + /// for that reason we serialize XID in 2 parts: + /// - lower: 32 least significant bits of 64bit XID OR 32bit XID + /// - upper: 32 most significant bits of 64bit XID + XidHelper xid_helper; + Coordination::read(xid_helper.parts.lower, buffer); + + /// go to end of the buffer and read extra information including second part of XID + auto buffer_position = buffer.getPosition(); + buffer.seek(length - sizeof(uint32_t), SEEK_CUR); + + using enum ZooKeeperLogSerializationVersion; + ZooKeeperLogSerializationVersion version = INITIAL; + + if (!buffer.eof()) + { + version = WITH_TIME; + readIntBinary(request_for_session->time, buffer); + } + else + request_for_session->time + = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + + if (!buffer.eof()) + { + version = WITH_ZXID_DIGEST; + + readIntBinary(request_for_session->zxid, buffer); + + chassert(!buffer.eof()); + + request_for_session->digest.emplace(); + readIntBinary(request_for_session->digest->version, buffer); + if (request_for_session->digest->version != KeeperStorageBase::DigestVersion::NO_DIGEST || !buffer.eof()) + readIntBinary(request_for_session->digest->value, buffer); + } + + if (!buffer.eof()) + { + version = WITH_XID_64; + Coordination::read(xid_helper.parts.upper, buffer); + } + + if (serialization_version) + *serialization_version = version; + + int64_t xid = xid_helper.xid; + + buffer.seek(buffer_position, SEEK_SET); static constexpr std::array non_cacheable_xids{ Coordination::WATCH_XID, Coordination::PING_XID, Coordination::AUTH_XID, Coordination::CLOSE_XID, + Coordination::CLOSE_XID_64, }; const bool should_cache @@ -266,44 +354,13 @@ std::shared_ptr IKeeperStateMachine::parse } } - Coordination::OpNum opnum; - Coordination::read(opnum, buffer); request_for_session->request = Coordination::ZooKeeperRequestFactory::instance().get(opnum); request_for_session->request->xid = xid; request_for_session->request->readImpl(buffer); - using enum ZooKeeperLogSerializationVersion; - ZooKeeperLogSerializationVersion version = INITIAL; - - if (!buffer.eof()) - { - version = WITH_TIME; - readIntBinary(request_for_session->time, buffer); - } - else - request_for_session->time - = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - - if (!buffer.eof()) - { - version = WITH_ZXID_DIGEST; - - readIntBinary(request_for_session->zxid, buffer); - - chassert(!buffer.eof()); - - request_for_session->digest.emplace(); - readIntBinary(request_for_session->digest->version, buffer); - if (request_for_session->digest->version != KeeperStorageBase::DigestVersion::NO_DIGEST || !buffer.eof()) - readIntBinary(request_for_session->digest->value, buffer); - } - - if (serialization_version) - *serialization_version = version; - if (should_cache && !final) { std::lock_guard lock(request_cache_mutex); @@ -777,14 +834,14 @@ static int bufferFromFile(LoggerPtr log, const std::string & path, nuraft::ptrput_raw(chunk, file_size); ::munmap(chunk, file_size); - int err = ::close(fd); + [[maybe_unused]] int err = ::close(fd); chassert(!err || errno == EINTR); return 0; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 03204036e7e..f1f16cfc653 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -35,6 +35,7 @@ public: INITIAL = 0, WITH_TIME = 1, WITH_ZXID_DIGEST = 2, + WITH_XID_64 = 3, }; /// lifetime of a parsed request is: @@ -45,7 +46,10 @@ public: /// /// final - whether it's the final time we will fetch the request so we can safely remove it from cache /// serialization_version - information about which fields were parsed from the buffer so we can modify the buffer accordingly - std::shared_ptr parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version = nullptr); + std::shared_ptr + parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version = nullptr); + + static nuraft::ptr getZooKeeperLogEntry(const KeeperStorageBase::RequestForSession & request_for_session); virtual bool preprocess(const KeeperStorageBase::RequestForSession & request_for_session) = 0; diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index 05290c04041..15f520154b1 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -339,7 +339,7 @@ void KeeperStateManager::save_state(const nuraft::srv_state & state) { auto buf = disk->writeFile(copy_lock_file); buf->finalize(); - disk->copyFile(server_state_file_name, *disk, old_path); + disk->copyFile(server_state_file_name, *disk, old_path, ReadSettings{}); disk->removeFile(copy_lock_file); disk->removeFile(old_path); } @@ -374,7 +374,7 @@ nuraft::ptr KeeperStateManager::read_state() { try { - auto read_buf = disk->readFile(path); + auto read_buf = disk->readFile(path, getReadSettings()); auto content_size = read_buf->getFileSize(); if (content_size == 0) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 69990d1661c..aecac7d672e 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -175,31 +175,54 @@ TYPED_TEST(CoordinationTest, BufferSerde) request->xid = 3; dynamic_cast(*request).path = "/path/value"; - DB::WriteBufferFromNuraftBuffer wbuf; - request->write(wbuf); - auto nuraft_buffer = wbuf.getBuffer(); - EXPECT_EQ(nuraft_buffer->size(), 28); + const auto test_serde = [&](bool use_xid_64) + { + size_t xid_size = use_xid_64 ? sizeof(int64_t) : sizeof(int32_t); + DB::WriteBufferFromNuraftBuffer wbuf; + request->write(wbuf, use_xid_64); + auto nuraft_buffer = wbuf.getBuffer(); + EXPECT_EQ(nuraft_buffer->size(), 24 + xid_size); - DB::ReadBufferFromNuraftBuffer rbuf(nuraft_buffer); + DB::ReadBufferFromNuraftBuffer rbuf(nuraft_buffer); - int32_t length; - Coordination::read(length, rbuf); - EXPECT_EQ(length + sizeof(length), nuraft_buffer->size()); + int32_t length; + Coordination::read(length, rbuf); + EXPECT_EQ(length + sizeof(length), nuraft_buffer->size()); - int32_t xid; - Coordination::read(xid, rbuf); - EXPECT_EQ(xid, request->xid); + int64_t xid = 0; + if (use_xid_64) + { + Coordination::read(xid, rbuf); + } + else + { + int32_t xid_32 = 0; + Coordination::read(xid_32, rbuf); + xid = xid_32; + } - Coordination::OpNum opnum; - Coordination::read(opnum, rbuf); + EXPECT_EQ(xid, request->xid); - Coordination::ZooKeeperRequestPtr request_read = Coordination::ZooKeeperRequestFactory::instance().get(opnum); - request_read->xid = xid; - request_read->readImpl(rbuf); + Coordination::OpNum opnum; + Coordination::read(opnum, rbuf); - EXPECT_EQ(request_read->getOpNum(), Coordination::OpNum::Get); - EXPECT_EQ(request_read->xid, 3); - EXPECT_EQ(dynamic_cast(*request_read).path, "/path/value"); + Coordination::ZooKeeperRequestPtr request_read = Coordination::ZooKeeperRequestFactory::instance().get(opnum); + request_read->xid = xid; + request_read->readImpl(rbuf); + + EXPECT_EQ(request_read->getOpNum(), Coordination::OpNum::Get); + EXPECT_EQ(request_read->xid, 3); + EXPECT_EQ(dynamic_cast(*request_read).path, "/path/value"); + }; + + { + SCOPED_TRACE("32bit XID"); + test_serde(/*use_xid_64=*/false); + } + { + SCOPED_TRACE("64bit XID"); + test_serde(/*use_xid_64=*/true); + } } template @@ -1798,23 +1821,14 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotBroken) EXPECT_THROW(manager.restoreFromLatestSnapshot(), DB::Exception); } -nuraft::ptr getBufferFromZKRequest(int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request) -{ - DB::WriteBufferFromNuraftBuffer buf; - DB::writeIntBinary(session_id, buf); - request->write(buf); - using namespace std::chrono; - auto time = duration_cast(system_clock::now().time_since_epoch()).count(); - DB::writeIntBinary(time, buf); - DB::writeIntBinary(zxid, buf); - DB::writeIntBinary(DB::KeeperMemoryStorage::DigestVersion::NO_DIGEST, buf); - return buf.getBuffer(); -} - nuraft::ptr getLogEntryFromZKRequest(size_t term, int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request) { - auto buffer = getBufferFromZKRequest(session_id, zxid, request); + DB::KeeperStorageBase::RequestForSession request_for_session; + request_for_session.session_id = session_id; + request_for_session.zxid = zxid; + request_for_session.request = request; + auto buffer = DB::IKeeperStateMachine::getZooKeeperLogEntry(request_for_session); return nuraft::cs_new(term, buffer); } diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index d560cb2c105..c7e0e9b7b37 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -818,6 +818,23 @@ Serializations Block::getSerializations() const return res; } +Serializations Block::getSerializations(const SerializationInfoByName & hints) const +{ + Serializations res; + res.reserve(data.size()); + + for (const auto & column : data) + { + auto it = hints.find(column.name); + if (it == hints.end()) + res.push_back(column.type->getDefaultSerialization()); + else + res.push_back(column.type->getSerialization(*it->second)); + } + + return res; +} + void convertToFullIfSparse(Block & block) { for (auto & column : block) diff --git a/src/Core/Block.h b/src/Core/Block.h index d998581a50f..841fb3fb663 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -99,6 +100,7 @@ public: NameMap getNamesToIndexesMap() const; Serializations getSerializations() const; + Serializations getSerializations(const SerializationInfoByName & hints) const; /// Returns number of rows from first column in block, not equal to nullptr. If no columns, returns 0. size_t rows() const; diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 4d33633b29d..07250a83236 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -97,7 +97,7 @@ void BaseExternalTable::parseStructureFromStructureField(const std::string & arg /*one_line=*/true, /*show_secrets=*/true, /*print_pretty_type_names=*/false, - /*always_quote_identifiers=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks)); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: expected column definition, got {}", child->formatForErrorMessage()); @@ -122,7 +122,7 @@ void BaseExternalTable::parseStructureFromTypesField(const std::string & argumen /*one_line=*/true, /*show_secrets=*/true, /*print_pretty_type_names=*/false, - /*always_quote_identifiers=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks)); } diff --git a/src/Core/FormatFactorySettingsDeclaration.h b/src/Core/FormatFactorySettingsDeclaration.h index 8b1f46901d4..8aa2997a5bd 100644 --- a/src/Core/FormatFactorySettingsDeclaration.h +++ b/src/Core/FormatFactorySettingsDeclaration.h @@ -257,8 +257,8 @@ M(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", "Overflow mode for Date, Date32, DateTime, DateTime64 types. Possible values: 'ignore', 'throw', 'saturate'.", 0) \ M(Bool, validate_experimental_and_suspicious_types_inside_nested_types, true, "Validate usage of experimental and suspicious types inside nested types like Array/Map/Tuple", 0) \ \ - M(Bool, output_format_always_quote_identifiers, false, "Always quote identifiers", 0) \ - M(IdentifierQuotingStyle, output_format_identifier_quoting_style, IdentifierQuotingStyle::Backticks, "Set the quoting style for identifiers", 0) \ + M(IdentifierQuotingRule, show_create_query_identifier_quoting_rule, IdentifierQuotingRule::WhenNecessary, "Set the quoting rule for identifiers in SHOW CREATE query", 0) \ + M(IdentifierQuotingStyle, show_create_query_identifier_quoting_style, IdentifierQuotingStyle::Backticks, "Set the quoting style for identifiers in SHOW CREATE query", 0) \ // End of FORMAT_FACTORY_SETTINGS diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 43c0bf6d77d..d98a4a42bb0 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -240,6 +240,7 @@ namespace ErrorCodes M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for some data formats.", 0) \ M(UInt64, output_format_compression_level, 3, "Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when inserting to table function `file`, `url`, `hdfs`, `s3`, and `azureBlobStorage`.", 0) \ M(UInt64, output_format_compression_zstd_window_log, 0, "Can be used when the output compression method is `zstd`. If greater than `0`, this setting explicitly sets compression window size (power of `2`) and enables a long-range mode for zstd compression.", 0) \ + M(Bool, enable_parsing_to_custom_serialization, true, "If true then data can be parsed directly to columns with custom serialization (e.g. Sparse) according to hints for serialization got from the table.", 0) \ \ M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6bb66039afb..5909ab6314c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -59,6 +59,7 @@ class WriteBuffer; M(CLASS_NAME, Double) \ M(CLASS_NAME, EscapingRule) \ M(CLASS_NAME, Float) \ + M(CLASS_NAME, IdentifierQuotingRule) \ M(CLASS_NAME, IdentifierQuotingStyle) \ M(CLASS_NAME, Int32) \ M(CLASS_NAME, Int64) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 7c3747382f5..6251b689602 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -67,10 +67,13 @@ static std::initializer_listsecond->add(*info); - } +void SerializationInfoByName::add(const String & name, const SerializationInfo & info) +{ + if (auto it = find(name); it != end()) + it->second->add(info); +} + +void SerializationInfoByName::remove(const SerializationInfoByName & other) +{ + for (const auto & [name, info] : other) + remove(name, *info); +} + +void SerializationInfoByName::remove(const String & name, const SerializationInfo & info) +{ + if (auto it = find(name); it != end()) + it->second->remove(info); +} + +SerializationInfoPtr SerializationInfoByName::tryGet(const String & name) const +{ + auto it = find(name); + return it == end() ? nullptr : it->second; +} + +MutableSerializationInfoPtr SerializationInfoByName::tryGet(const String & name) +{ + auto it = find(name); + return it == end() ? nullptr : it->second; } void SerializationInfoByName::replaceData(const SerializationInfoByName & other) @@ -224,6 +262,12 @@ void SerializationInfoByName::replaceData(const SerializationInfoByName & other) } } +ISerialization::Kind SerializationInfoByName::getKind(const String & column_name) const +{ + auto it = find(column_name); + return it != end() ? it->second->getKind() : ISerialization::Kind::DEFAULT; +} + void SerializationInfoByName::writeJSON(WriteBuffer & out) const { Poco::JSON::Object object; diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index 5a900a5521c..c30e50ab12c 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -39,6 +39,7 @@ public: void add(const IColumn & column); void add(const Data & other); + void remove(const Data & other); void addDefaults(size_t length); }; @@ -52,6 +53,7 @@ public: virtual void add(const IColumn & column); virtual void add(const SerializationInfo & other); + virtual void remove(const SerializationInfo & other); virtual void addDefaults(size_t length); virtual void replaceData(const SerializationInfo & other); @@ -99,6 +101,14 @@ public: void add(const Block & block); void add(const SerializationInfoByName & other); + void add(const String & name, const SerializationInfo & info); + + void remove(const SerializationInfoByName & other); + void remove(const String & name, const SerializationInfo & info); + + SerializationInfoPtr tryGet(const String & name) const; + MutableSerializationInfoPtr tryGet(const String & name); + ISerialization::Kind getKind(const String & column_name) const; /// Takes data from @other, but keeps current serialization kinds. /// If column exists in @other infos, but not in current infos, diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index cd65b865248..b7449be3cc5 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -10,6 +10,7 @@ namespace ErrorCodes { extern const int CORRUPTED_DATA; extern const int THERE_IS_NO_COLUMN; + extern const int NOT_IMPLEMENTED; } SerializationInfoTuple::SerializationInfoTuple( @@ -68,6 +69,19 @@ void SerializationInfoTuple::add(const SerializationInfo & other) } } +void SerializationInfoTuple::remove(const SerializationInfo & other) +{ + if (!structureEquals(other)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot remove from serialization info different structure"); + + SerializationInfo::remove(other); + const auto & other_elems = assert_cast(other).elems; + chassert(elems.size() == other_elems.size()); + + for (size_t i = 0; i < elems.size(); ++i) + elems[i]->remove(*other_elems[i]); +} + void SerializationInfoTuple::addDefaults(size_t length) { SerializationInfo::addDefaults(length); diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.h b/src/DataTypes/Serializations/SerializationInfoTuple.h index a9f3bdb6c6e..a6b9c89166f 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.h +++ b/src/DataTypes/Serializations/SerializationInfoTuple.h @@ -15,6 +15,7 @@ public: void add(const IColumn & column) override; void add(const SerializationInfo & other) override; + void remove(const SerializationInfo & other) override; void addDefaults(size_t length) override; void replaceData(const SerializationInfo & other) override; diff --git a/src/DataTypes/Serializations/SerializationSparse.cpp b/src/DataTypes/Serializations/SerializationSparse.cpp index 73488d308bb..327d1f23cca 100644 --- a/src/DataTypes/Serializations/SerializationSparse.cpp +++ b/src/DataTypes/Serializations/SerializationSparse.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { - extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; } @@ -313,15 +312,35 @@ void SerializationSparse::deserializeBinary(Field & field, ReadBuffer & istr, co nested->deserializeBinary(field, istr, settings); } +template +void SerializationSparse::deserialize(IColumn & column, Reader && reader) const +{ + auto & column_sparse = assert_cast(column); + auto & values = column_sparse.getValuesColumn(); + size_t old_size = column_sparse.size(); + + /// It just increments the size of column. + column_sparse.insertDefault(); + reader(column_sparse.getValuesColumn()); + + if (values.isDefaultAt(values.size() - 1)) + values.popBack(1); + else + column_sparse.getOffsetsData().push_back(old_size); +} + void SerializationSparse::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { const auto & column_sparse = assert_cast(column); nested->serializeBinary(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeBinary(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeBinary' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeBinary(nested_column, istr, settings); + }); } void SerializationSparse::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -330,9 +349,12 @@ void SerializationSparse::serializeTextEscaped(const IColumn & column, size_t ro nested->serializeTextEscaped(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeTextEscaped(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeTextEscaped' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeTextEscaped(nested_column, istr, settings); + }); } void SerializationSparse::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -341,9 +363,12 @@ void SerializationSparse::serializeTextQuoted(const IColumn & column, size_t row nested->serializeTextQuoted(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeTextQuoted(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeTextQuoted' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeTextQuoted(nested_column, istr, settings); + }); } void SerializationSparse::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -352,9 +377,12 @@ void SerializationSparse::serializeTextCSV(const IColumn & column, size_t row_nu nested->serializeTextCSV(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeTextCSV(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeTextCSV' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeTextCSV(nested_column, istr, settings); + }); } void SerializationSparse::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -363,9 +391,12 @@ void SerializationSparse::serializeText(const IColumn & column, size_t row_num, nested->serializeText(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeWholeText(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeWholeText' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeWholeText(nested_column, istr, settings); + }); } void SerializationSparse::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const @@ -374,9 +405,12 @@ void SerializationSparse::serializeTextJSON(const IColumn & column, size_t row_n nested->serializeTextJSON(column_sparse.getValuesColumn(), column_sparse.getValueIndex(row_num), ostr, settings); } -void SerializationSparse::deserializeTextJSON(IColumn &, ReadBuffer &, const FormatSettings &) const +void SerializationSparse::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'deserializeTextJSON' is not implemented for SerializationSparse"); + deserialize(column, [&](auto & nested_column) + { + nested->deserializeTextJSON(nested_column, istr, settings); + }); } void SerializationSparse::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/Serializations/SerializationSparse.h b/src/DataTypes/Serializations/SerializationSparse.h index a55856bacf0..b31a006639b 100644 --- a/src/DataTypes/Serializations/SerializationSparse.h +++ b/src/DataTypes/Serializations/SerializationSparse.h @@ -99,6 +99,9 @@ private: ColumnPtr create(const ColumnPtr & prev) const override; }; + template + void deserialize(IColumn & column, Reader && reader) const; + SerializationPtr nested; }; diff --git a/src/Dictionaries/ExternalQueryBuilder.cpp b/src/Dictionaries/ExternalQueryBuilder.cpp index a31301cd0f3..0b8b888fcbb 100644 --- a/src/Dictionaries/ExternalQueryBuilder.cpp +++ b/src/Dictionaries/ExternalQueryBuilder.cpp @@ -53,10 +53,6 @@ void ExternalQueryBuilder::writeQuoted(const std::string & s, WriteBuffer & out) { switch (quoting_style) { - case IdentifierQuotingStyle::None: - writeString(s, out); - break; - case IdentifierQuotingStyle::Backticks: writeBackQuotedString(s, out); break; diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index e96bdc4ac55..2b53b59bdac 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -769,7 +769,7 @@ private: if (this == &rhs) return *this; - int err = ::close(fd); + [[maybe_unused]] int err = ::close(fd); chassert(!err || errno == EINTR); fd = rhs.fd; @@ -780,7 +780,7 @@ private: { if (fd != -1) { - int err = close(fd); + [[maybe_unused]] int err = close(fd); chassert(!err || errno == EINTR); } } diff --git a/src/Disks/DiskEncrypted.cpp b/src/Disks/DiskEncrypted.cpp index 68fd9012857..6a91ac2d0fc 100644 --- a/src/Disks/DiskEncrypted.cpp +++ b/src/Disks/DiskEncrypted.cpp @@ -392,7 +392,7 @@ size_t DiskEncrypted::getFileSize(const String & path) const UInt128 DiskEncrypted::getEncryptedFileIV(const String & path) const { auto wrapped_path = wrappedPath(path); - auto read_buffer = delegate->readFile(wrapped_path, ReadSettings().adjustBufferSize(FileEncryption::Header::kSize)); + auto read_buffer = delegate->readFile(wrapped_path, getReadSettings().adjustBufferSize(FileEncryption::Header::kSize)); if (read_buffer->eof()) return 0; auto header = readHeader(*read_buffer); diff --git a/src/Disks/DiskEncryptedTransaction.cpp b/src/Disks/DiskEncryptedTransaction.cpp index daeab7aae6c..827b42cf0cd 100644 --- a/src/Disks/DiskEncryptedTransaction.cpp +++ b/src/Disks/DiskEncryptedTransaction.cpp @@ -78,7 +78,7 @@ std::unique_ptr DiskEncryptedTransaction::writeFile( // if (old_file_size) { /// Append mode: we continue to use the same header. - auto read_buffer = delegate_disk->readFile(wrapped_path, ReadSettings().adjustBufferSize(FileEncryption::Header::kSize)); + auto read_buffer = delegate_disk->readFile(wrapped_path, getReadSettings().adjustBufferSize(FileEncryption::Header::kSize)); header = readHeader(*read_buffer); key = current_settings.findKeyByFingerprint(header.key_fingerprint, path); } diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 4bbefad5290..baeb2197aad 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -196,6 +196,7 @@ void IDisk::checkAccessImpl(const String & path) try { const std::string_view payload("test", 4); + const auto read_settings = getReadSettings(); /// write { @@ -215,7 +216,7 @@ try /// read { - auto file = readFile(path); + auto file = readFile(path, read_settings); String buf(payload.size(), '0'); file->readStrict(buf.data(), buf.size()); if (buf != payload) @@ -227,7 +228,7 @@ try /// read with offset { - auto file = readFile(path); + auto file = readFile(path, read_settings); auto offset = 2; String buf(payload.size() - offset, '0'); file->seek(offset, 0); diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 78d5f37e3a7..fbe1cad9912 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -209,7 +209,7 @@ public: const String & from_file_path, IDisk & to_disk, const String & to_file_path, - const ReadSettings & read_settings = {}, + const ReadSettings & read_settings, const WriteSettings & write_settings = {}, const std::function & cancellation_hook = {}); @@ -219,7 +219,7 @@ public: /// Open the file for read and return ReadBufferFromFileBase object. virtual std::unique_ptr readFile( /// NOLINT const String & path, - const ReadSettings & settings = ReadSettings{}, + const ReadSettings & settings, std::optional read_hint = {}, std::optional file_size = {}) const = 0; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index bc90b05e64d..15a0bfb9ac1 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -47,13 +47,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index efcdbfebabf..03b013c2eed 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -33,13 +33,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index 5c45a258806..602a61c9a99 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -155,7 +155,7 @@ public: const String & from_file_path, IDisk & to_disk, const String & to_file_path, - const ReadSettings & read_settings = {}, + const ReadSettings & read_settings, const WriteSettings & write_settings = {}, const std::function & cancellation_hook = {} ) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index 701c08b9a14..0eb58cba4bd 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -73,7 +73,7 @@ int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStora if (!object_storage->exists(object)) return version; - auto buf = object_storage->readObject(object); + auto buf = object_storage->readObject(object, ReadSettings{}); readIntText(version, *buf); return version; diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index 0cb31eb8b8b..21c2b0635ca 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -65,13 +65,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index f3c587a1188..72f6d150d34 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -146,14 +146,14 @@ public: /// Read single object virtual std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const = 0; /// Read multiple objects with common prefix virtual std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const = 0; diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index 564d49bf876..155359ce663 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -30,13 +30,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index 493470982cb..e5240334fa8 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -66,7 +66,7 @@ DirectoryIteratorPtr MetadataStorageFromDisk::iterateDirectory(const std::string std::string MetadataStorageFromDisk::readFileToString(const std::string & path) const { - auto buf = disk->readFile(path); + auto buf = disk->readFile(path, ReadSettings{}); std::string result; readStringUntilEOF(result, *buf); return result; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp index 79d1f4a1f7c..935d126f3b8 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp @@ -275,7 +275,7 @@ void WriteFileOperation::execute(std::unique_lock &) if (disk.exists(path)) { existed = true; - auto buf = disk.readFile(path); + auto buf = disk.readFile(path, ReadSettings{}); readStringUntilEOF(prev_data, *buf); } auto buf = disk.writeFile(path); diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp index bfd203ef2e0..aaccc0eaaf0 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp @@ -150,7 +150,7 @@ std::unique_ptr MetadataStorageFromPlainObjectStorageMo if (validate_content) { std::string data; - auto read_buf = object_storage->readObject(metadata_object); + auto read_buf = object_storage->readObject(metadata_object, ReadSettings{}); readStringUntilEOF(data, *read_buf); if (data != path_from) throw Exception( diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp index 39b11d9a3e3..0a133d1042a 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp @@ -51,7 +51,7 @@ std::shared_ptr loadPathPrefixMap(const std::string & metadata_ LoggerPtr log = getLogger("MetadataStorageFromPlainObjectStorage"); - ReadSettings settings; + auto settings = getReadSettings(); settings.enable_filesystem_cache = false; settings.remote_fs_method = RemoteFSReadMethod::read; settings.remote_fs_buffer_size = 1024; /// These files are small. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index d786a6b37f3..b99867d8663 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -85,13 +85,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index ab357d6f50d..9f7f7c137e2 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -35,13 +35,13 @@ public: std::unique_ptr readObject( /// NOLINT const StoredObject & object, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT const StoredObjects & objects, - const ReadSettings & read_settings = ReadSettings{}, + const ReadSettings & read_settings, std::optional read_hint = {}, std::optional file_size = {}) const override; diff --git a/src/Disks/tests/gtest_disk.cpp b/src/Disks/tests/gtest_disk.cpp index 87882147578..374dcc1d474 100644 --- a/src/Disks/tests/gtest_disk.cpp +++ b/src/Disks/tests/gtest_disk.cpp @@ -49,7 +49,7 @@ TEST_F(DiskTest, writeFile) String data; { - std::unique_ptr in = disk->readFile("test_file"); + std::unique_ptr in = disk->readFile("test_file", DB::getReadSettings()); readString(data, *in); } @@ -65,10 +65,12 @@ TEST_F(DiskTest, readFile) writeString("test data", *out); } + auto read_settings = DB::getReadSettings(); + // Test SEEK_SET { String buf(4, '0'); - std::unique_ptr in = disk->readFile("test_file"); + std::unique_ptr in = disk->readFile("test_file", read_settings); in->seek(5, SEEK_SET); @@ -78,7 +80,7 @@ TEST_F(DiskTest, readFile) // Test SEEK_CUR { - std::unique_ptr in = disk->readFile("test_file"); + std::unique_ptr in = disk->readFile("test_file", read_settings); String buf(4, '0'); in->readStrict(buf.data(), 4); diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 74c33e3a1d8..20426e8eccb 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -3322,124 +3322,138 @@ namespace } } + /// Complex case: one or more columns are serialized as a nested message. for (const auto & [field_descriptor, suffix] : field_descriptors_with_suffixes) { - if (!suffix.empty()) + if (suffix.empty()) + continue; + + std::vector nested_column_indices; + std::vector nested_column_names; + nested_column_indices.reserve(num_columns - used_column_indices.size()); + nested_column_names.reserve(num_columns - used_column_indices.size()); + nested_column_indices.push_back(column_idx); + nested_column_names.push_back(suffix); + + for (size_t j : collections::range(column_idx + 1, num_columns)) { - /// Complex case: one or more columns are serialized as a nested message. - std::vector nested_column_indices; - std::vector nested_column_names; - nested_column_indices.reserve(num_columns - used_column_indices.size()); - nested_column_names.reserve(num_columns - used_column_indices.size()); - nested_column_indices.push_back(column_idx); - nested_column_names.push_back(suffix); + if (used_column_indices_sorted.count(j)) + continue; + std::string_view other_suffix; + if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, other_suffix)) + continue; + nested_column_indices.push_back(j); + nested_column_names.push_back(other_suffix); + } - for (size_t j : collections::range(column_idx + 1, num_columns)) + DataTypes nested_data_types; + nested_data_types.reserve(nested_column_indices.size()); + for (size_t j : nested_column_indices) + nested_data_types.push_back(data_types[j]); + + /// Now we have up to `nested_message_column_names.size()` columns + /// which can be serialized as one or many nested message(s) + + /// If the field is repeated, and ALL matching columns are array, we serialize as an array of nested messages. + /// Otherwise, we first try to serialize those columns as one nested message, + /// then, if failed, as an array of nested messages (on condition if those columns are array). + + bool repeated_field_matching_nested_columns_are_all_arrays = false; + bool repeated_field_matching_nested_columns_have_some_arrays = false; + if (field_descriptor->is_repeated()) + { + repeated_field_matching_nested_columns_are_all_arrays = true; + for (const auto & nested_data_type : nested_data_types) { - if (used_column_indices_sorted.count(j)) - continue; - std::string_view other_suffix; - if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, other_suffix)) - continue; - nested_column_indices.push_back(j); - nested_column_names.push_back(other_suffix); + if (nested_data_type->getTypeId() == TypeIndex::Array) + repeated_field_matching_nested_columns_have_some_arrays = true; + else + repeated_field_matching_nested_columns_are_all_arrays = false; + } + } + + std::vector used_column_indices_in_nested; + auto attempt_build_serializer = [&](const DataTypes & passed_nested_data_types) + { + return buildMessageSerializerImpl( + nested_column_names.size(), + nested_column_names.data(), + passed_nested_data_types.data(), + *field_descriptor->message_type(), + /* with_length_delimiter = */ false, + google_wrappers_special_treatment, + field_descriptor, + used_column_indices_in_nested, + /* columns_are_reordered_outside = */ true, + /* check_nested_while_filling_missing_columns = */ false); + + /// `columns_are_reordered_outside` is true because column indices are + /// going to be transformed and then written to the outer message, + /// see next calls to add_field_serializer() further below. + }; + + auto attempt_unwrap_and_build_array_serializer = [&]() + { + DataTypes unwrapped_nested_data_types; + unwrapped_nested_data_types.reserve(nested_data_types.size()); + + for (DataTypePtr & dt : nested_data_types) + unwrapped_nested_data_types.push_back(assert_cast(*dt).getNestedType()); + + if (auto serializer = attempt_build_serializer(unwrapped_nested_data_types)) + { + std::vector column_names_used; + column_names_used.reserve(used_column_indices_in_nested.size()); + for (const size_t i : used_column_indices_in_nested) + column_names_used.emplace_back(nested_column_names[i]); + + auto array_serializer = std::make_unique( + std::move(column_names_used), field_descriptor, std::move(serializer), get_root_desc_function); + + transformColumnIndices(used_column_indices_in_nested, nested_column_indices); + add_field_serializer(column_name,std::move(used_column_indices_in_nested), *field_descriptor, std::move(array_serializer)); + + return true; } - DataTypes nested_data_types; - nested_data_types.reserve(nested_column_indices.size()); - for (size_t j : nested_column_indices) - nested_data_types.push_back(data_types[j]); + return false; + }; - /// Now we have up to `nested_message_column_names.size()` columns - /// which can be serialized as a nested message. + /// if the protobuf field has the repeated label, + /// for ALL matching nested cols, since they are all of type array + /// try as ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages + if (repeated_field_matching_nested_columns_are_all_arrays) + { + if (attempt_unwrap_and_build_array_serializer()) + break; + } - /// We will try to serialize those columns as one nested message, - /// then, if failed, as an array of nested messages (on condition if those columns are array). - bool has_fallback_to_array_of_nested_messages = false; - if (field_descriptor->is_repeated()) + /// for ALL matching nested cols + /// try as ProtobufSerializerMessage + try + { + if (auto serializer = attempt_build_serializer(nested_data_types)) { - bool has_arrays - = boost::range::find_if( - nested_data_types, [](const DataTypePtr & dt) { return (dt->getTypeId() == TypeIndex::Array); }) - != nested_data_types.end(); - if (has_arrays) - has_fallback_to_array_of_nested_messages = true; + transformColumnIndices(used_column_indices_in_nested, nested_column_indices); + add_field_serializer(column_name,std::move(used_column_indices_in_nested), *field_descriptor, std::move(serializer)); + break; } + } - /// Try to serialize those columns as one nested message. - try - { - std::vector used_column_indices_in_nested; - auto nested_message_serializer = buildMessageSerializerImpl( - nested_column_names.size(), - nested_column_names.data(), - nested_data_types.data(), - *field_descriptor->message_type(), - /* with_length_delimiter = */ false, - google_wrappers_special_treatment, - field_descriptor, - used_column_indices_in_nested, - /* columns_are_reordered_outside = */ true, - /* check_nested_while_filling_missing_columns = */ false); + catch (Exception & e) + { + if ((e.code() != ErrorCodes::PROTOBUF_FIELD_NOT_REPEATED) || !repeated_field_matching_nested_columns_have_some_arrays) + throw; + } - /// `columns_are_reordered_outside` is true because column indices are - /// going to be transformed and then written to the outer message, - /// see add_field_serializer() below. - - if (nested_message_serializer) - { - transformColumnIndices(used_column_indices_in_nested, nested_column_indices); - add_field_serializer( - column_name, - std::move(used_column_indices_in_nested), - *field_descriptor, - std::move(nested_message_serializer)); - break; - } - } - catch (Exception & e) - { - if ((e.code() != ErrorCodes::PROTOBUF_FIELD_NOT_REPEATED) || !has_fallback_to_array_of_nested_messages) - throw; - } - - if (has_fallback_to_array_of_nested_messages) - { - /// Try to serialize those columns as an array of nested messages. - removeNonArrayElements(nested_data_types, nested_column_names, nested_column_indices); - for (DataTypePtr & dt : nested_data_types) - dt = assert_cast(*dt).getNestedType(); - - std::vector used_column_indices_in_nested; - auto nested_message_serializer = buildMessageSerializerImpl( - nested_column_names.size(), - nested_column_names.data(), - nested_data_types.data(), - *field_descriptor->message_type(), - /* with_length_delimiter = */ false, - google_wrappers_special_treatment, - field_descriptor, - used_column_indices_in_nested, - /* columns_are_reordered_outside = */ true, - /* check_nested_while_filling_missing_columns = */ false); - - /// `columns_are_reordered_outside` is true because column indices are - /// going to be transformed and then written to the outer message, - /// see add_field_serializer() below. - - if (nested_message_serializer) - { - std::vector column_names_used; - column_names_used.reserve(used_column_indices_in_nested.size()); - for (size_t i : used_column_indices_in_nested) - column_names_used.emplace_back(nested_column_names[i]); - auto field_serializer = std::make_unique( - std::move(column_names_used), field_descriptor, std::move(nested_message_serializer), get_root_desc_function); - transformColumnIndices(used_column_indices_in_nested, nested_column_indices); - add_field_serializer(column_name, std::move(used_column_indices_in_nested), *field_descriptor, std::move(field_serializer)); - break; - } - } + /// if the protobuf field has the repeated label, + /// only for the SUBSET of matching nested cols that are of type Array, + /// try as ProtobufSerializerFlattenedNestedAsArrayOfNestedMessages + if (repeated_field_matching_nested_columns_have_some_arrays) + { + removeNonArrayElements(nested_data_types, nested_column_names, nested_column_indices); + if (attempt_unwrap_and_build_array_serializer()) + break; } } } diff --git a/src/Functions/FunctionsStringHashFixedString.cpp b/src/Functions/FunctionsStringHashFixedString.cpp index 9474fe2629e..d8f3a3f7c09 100644 --- a/src/Functions/FunctionsStringHashFixedString.cpp +++ b/src/Functions/FunctionsStringHashFixedString.cpp @@ -14,10 +14,11 @@ #endif #if USE_SSL +# include # include # include +# include # include -# include #endif /// Instatiating only the functions that require FunctionStringHashFixedString in a separate file @@ -175,6 +176,23 @@ struct SHA512Impl256 EVP_MD_CTX_destroy(md_ctx); } }; + +struct RIPEMD160Impl +{ + static constexpr auto name = "RIPEMD160"; + enum + { + length = RIPEMD160_DIGEST_LENGTH + }; + + static void apply(const char * begin, const size_t size, unsigned char * out_char_data) + { + RIPEMD160_CTX ctx; + RIPEMD160_Init(&ctx); + RIPEMD160_Update(&ctx, reinterpret_cast(begin), size); + RIPEMD160_Final(out_char_data, &ctx); + } +}; #endif #if USE_BLAKE3 @@ -271,8 +289,7 @@ public: chars_to.resize(input_rows_count * Impl::length); for (size_t i = 0; i < input_rows_count; ++i) { - Impl::apply( - reinterpret_cast(&data[i]), length, reinterpret_cast(&chars_to[i * Impl::length])); + Impl::apply(reinterpret_cast(&data[i]), length, reinterpret_cast(&chars_to[i * Impl::length])); } return col_to; } @@ -297,7 +314,22 @@ REGISTER_FUNCTION(HashFixedStrings) using FunctionSHA384 = FunctionStringHashFixedString; using FunctionSHA512 = FunctionStringHashFixedString; using FunctionSHA512_256 = FunctionStringHashFixedString; + using FunctionRIPEMD160 = FunctionStringHashFixedString; + factory.registerFunction(FunctionDocumentation{ + .description = R"(Calculates the RIPEMD-160 hash of the given string.)", + .syntax = "SELECT RIPEMD160(s);", + .arguments = {{"s", "The input [String](../../sql-reference/data-types/string.md)."}}, + .returned_value + = "The RIPEMD160 hash of the given input string returned as a [FixedString(20)](../../sql-reference/data-types/fixedstring.md).", + .examples + = {{"", + "SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'));", + R"( +┌─HEX(RIPEMD160('The quick brown fox jumps over the lazy dog'))─┐ +│ 37F332F68DB77BD9D7EDD4969571AD671CF9DD3B │ +└───────────────────────────────────────────────────────────────┘ + )"}}}); factory.registerFunction(FunctionDocumentation{ .description = R"(Calculates the MD4 hash of the given string.)", .syntax = "SELECT MD4(s);", @@ -416,16 +448,15 @@ REGISTER_FUNCTION(HashFixedStrings) # if USE_BLAKE3 using FunctionBLAKE3 = FunctionStringHashFixedString; - factory.registerFunction( - FunctionDocumentation{ - .description = R"( + factory.registerFunction(FunctionDocumentation{ + .description = R"( Calculates BLAKE3 hash string and returns the resulting set of bytes as FixedString. This cryptographic hash-function is integrated into ClickHouse with BLAKE3 Rust library. The function is rather fast and shows approximately two times faster performance compared to SHA-2, while generating hashes of the same length as SHA-256. It returns a BLAKE3 hash as a byte array with type FixedString(32). )", - .examples{{"hash", "SELECT hex(BLAKE3('ABC'))", ""}}, - .categories{"Hash"}}); + .examples{{"hash", "SELECT hex(BLAKE3('ABC'))", ""}}, + .categories{"Hash"}}); # endif } #endif diff --git a/src/Functions/ReplaceStringImpl.h b/src/Functions/ReplaceStringImpl.h index 7c56d657b3e..9fa740d597d 100644 --- a/src/Functions/ReplaceStringImpl.h +++ b/src/Functions/ReplaceStringImpl.h @@ -8,11 +8,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int ARGUMENT_OUT_OF_BOUND; -} - struct ReplaceStringTraits { enum class Replace : uint8_t @@ -39,7 +34,11 @@ struct ReplaceStringImpl size_t input_rows_count) { if (needle.empty()) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name); + { + res_data.assign(haystack_data.begin(), haystack_data.end()); + res_offsets.assign(haystack_offsets.begin(), haystack_offsets.end()); + return; + } const UInt8 * const begin = haystack_data.data(); const UInt8 * const end = haystack_data.data() + haystack_data.size(); @@ -145,34 +144,34 @@ struct ReplaceStringImpl const auto * const cur_needle_data = &needle_data[prev_needle_offset]; const size_t cur_needle_length = needle_offsets[i] - prev_needle_offset - 1; - if (cur_needle_length == 0) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name); - - /// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row - StdLibASCIIStringSearcher searcher(cur_needle_data, cur_needle_length); - const auto * last_match = static_cast(nullptr); const auto * start_pos = cur_haystack_data; const auto * const cur_haystack_end = cur_haystack_data + cur_haystack_length; - while (start_pos < cur_haystack_end) + if (cur_needle_length) { - if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end) + /// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row + StdLibASCIIStringSearcher searcher(cur_needle_data, cur_needle_length); + + while (start_pos < cur_haystack_end) { - /// Copy prefix before match - copyToOutput(start_pos, match - start_pos, res_data, res_offset); + if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end) + { + /// Copy prefix before match + copyToOutput(start_pos, match - start_pos, res_data, res_offset); - /// Insert replacement for match - copyToOutput(replacement.data(), replacement.size(), res_data, res_offset); + /// Insert replacement for match + copyToOutput(replacement.data(), replacement.size(), res_data, res_offset); - last_match = match; - start_pos = match + cur_needle_length; + last_match = match; + start_pos = match + cur_needle_length; - if constexpr (replace == ReplaceStringTraits::Replace::First) + if constexpr (replace == ReplaceStringTraits::Replace::First) + break; + } + else break; } - else - break; } /// Copy suffix after last match @@ -200,7 +199,11 @@ struct ReplaceStringImpl chassert(haystack_offsets.size() == replacement_offsets.size()); if (needle.empty()) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name); + { + res_data.assign(haystack_data.begin(), haystack_data.end()); + res_offsets.assign(haystack_offsets.begin(), haystack_offsets.end()); + return; + } res_data.reserve(haystack_data.size()); res_offsets.resize(input_rows_count); @@ -291,36 +294,35 @@ struct ReplaceStringImpl const auto * const cur_replacement_data = &replacement_data[prev_replacement_offset]; const size_t cur_replacement_length = replacement_offsets[i] - prev_replacement_offset - 1; - if (cur_needle_length == 0) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name); - - /// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row - StdLibASCIIStringSearcher searcher(cur_needle_data, cur_needle_length); - const auto * last_match = static_cast(nullptr); const auto * start_pos = cur_haystack_data; const auto * const cur_haystack_end = cur_haystack_data + cur_haystack_length; - while (start_pos < cur_haystack_end) + if (cur_needle_length) { - if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end) + /// Using "slow" "stdlib searcher instead of Volnitsky because there is a different pattern in each row + StdLibASCIIStringSearcher searcher(cur_needle_data, cur_needle_length); + + while (start_pos < cur_haystack_end) { - /// Copy prefix before match - copyToOutput(start_pos, match - start_pos, res_data, res_offset); + if (const auto * const match = searcher.search(start_pos, cur_haystack_end); match != cur_haystack_end) + { + /// Copy prefix before match + copyToOutput(start_pos, match - start_pos, res_data, res_offset); - /// Insert replacement for match - copyToOutput(cur_replacement_data, cur_replacement_length, res_data, res_offset); + /// Insert replacement for match + copyToOutput(cur_replacement_data, cur_replacement_length, res_data, res_offset); - last_match = match; - start_pos = match + cur_needle_length; + last_match = match; + start_pos = match + cur_needle_length; - if constexpr (replace == ReplaceStringTraits::Replace::First) + if constexpr (replace == ReplaceStringTraits::Replace::First) + break; + } + else break; } - else - break; } - /// Copy suffix after last match size_t bytes = (last_match == nullptr) ? (cur_haystack_end - cur_haystack_data + 1) : (cur_haystack_end - last_match - cur_needle_length + 1); @@ -346,7 +348,21 @@ struct ReplaceStringImpl size_t input_rows_count) { if (needle.empty()) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name); + { + chassert(input_rows_count == haystack_data.size() / n); + /// Since ColumnFixedString does not have a zero byte at the end, while ColumnString does, + /// we need to split haystack_data into strings of length n, add 1 zero byte to the end of each string + /// and then copy to res_data, ref: ColumnString.h and ColumnFixedString.h + res_data.reserve(haystack_data.size() + input_rows_count); + res_offsets.resize(input_rows_count); + for (size_t i = 0; i < input_rows_count; ++i) + { + res_data.insert(res_data.end(), haystack_data.begin() + i * n, haystack_data.begin() + (i + 1) * n); + res_data.push_back(0); + res_offsets[i] = (i + 1) * n + 1; + } + return; + } const UInt8 * const begin = haystack_data.data(); const UInt8 * const end = haystack_data.data() + haystack_data.size(); diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index b3b0413f3be..82ad874e84a 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -18,7 +18,7 @@ ColumnPtr ArrayAllImpl::execute(const ColumnArray & array, ColumnPtr mapped) const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result of the lambda is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) return DataTypeUInt8().createColumnConst(array.size(), 1u); diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index 019f07e8e6a..398ff5941d3 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -35,7 +35,7 @@ struct ArrayCountImpl const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) { diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index d59425338a5..2c8a7648088 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -18,7 +18,7 @@ ColumnPtr ArrayExistsImpl::execute(const ColumnArray & array, ColumnPtr mapped) const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result of the lambda is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) { diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 618abd93845..72878edfe79 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -18,7 +18,7 @@ ColumnPtr ArrayFilterImpl::execute(const ColumnArray & array, ColumnPtr mapped) const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result of the lambda is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) return array.clone(); diff --git a/src/Functions/array/arrayFirstLast.cpp b/src/Functions/array/arrayFirstLast.cpp index 055c2a0f79a..e54d47a3f97 100644 --- a/src/Functions/array/arrayFirstLast.cpp +++ b/src/Functions/array/arrayFirstLast.cpp @@ -59,7 +59,7 @@ struct ArrayFirstLastImpl const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result of the lambda is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) { diff --git a/src/Functions/array/arrayFirstLastIndex.cpp b/src/Functions/array/arrayFirstLastIndex.cpp index 858b5ffac5a..54c9cb487de 100644 --- a/src/Functions/array/arrayFirstLastIndex.cpp +++ b/src/Functions/array/arrayFirstLastIndex.cpp @@ -39,7 +39,7 @@ struct ArrayFirstLastIndexImpl const auto * column_filter_const = checkAndGetColumnConst(&*mapped); if (!column_filter_const) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column"); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type of filter column: {}; The result of the lambda is expected to be a UInt8", mapped->getDataType()); if (column_filter_const->getValue()) { diff --git a/src/Functions/getSetting.cpp b/src/Functions/getSetting.cpp index aed6b2119e4..0ec2a7b11f3 100644 --- a/src/Functions/getSetting.cpp +++ b/src/Functions/getSetting.cpp @@ -19,11 +19,18 @@ namespace ErrorCodes namespace { +enum class ErrorHandlingMode : uint8_t +{ + Exception, /// Raise exception if setting not found (getSetting()) + Default, /// Return default value if setting not found (getSettingOrDefault()) +}; + /// Get the value of a setting. +template class FunctionGetSetting : public IFunction, WithContext { public: - static constexpr auto name = "getSetting"; + static constexpr auto name = (mode == ErrorHandlingMode::Exception) ? "getSetting" : "getSettingOrDefault"; static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } explicit FunctionGetSetting(ContextPtr context_) : WithContext(context_) {} @@ -31,8 +38,8 @@ public: String getName() const override { return name; } bool isDeterministic() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - size_t getNumberOfArguments() const override { return 1; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + size_t getNumberOfArguments() const override { return (mode == ErrorHandlingMode::Default) ? 2 : 1 ; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 1}; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -60,7 +67,21 @@ private: String{name}); std::string_view setting_name{column->getDataAt(0).toView()}; - return getContext()->getSettingsRef().get(setting_name); + Field setting_value; + if constexpr (mode == ErrorHandlingMode::Exception) + setting_value = getContext()->getSettingsRef().get(setting_name); + else + { + const auto * default_value_column = arguments[1].column.get(); + if (!default_value_column || !(isColumnConst(*default_value_column))) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The 2nd argument of function {} should be a constant with the default value of a setting", String{name}); + } + if (!getContext()->getSettingsRef().tryGet(setting_name, setting_value)) + setting_value = (*default_value_column)[0]; + } + return setting_value; } }; @@ -68,7 +89,35 @@ private: REGISTER_FUNCTION(GetSetting) { - factory.registerFunction(); + factory.registerFunction>(FunctionDocumentation{ + .description = R"( +Returns the current value of a custom setting. +)", + .syntax = "getSetting('custom_setting')", + .arguments = { + {"custom_setting", "The setting name. Type: String."} + }, + .returned_value = "The setting's current value.", + .examples = { + {"getSetting", "SET custom_a = 123; SELECT getSetting('custom_a');", "123"}, + }, + .categories{"Other"}}, FunctionFactory::Case::Sensitive); + factory.registerFunction>(FunctionDocumentation{ + .description = R"( +Returns the current value of a custom setting or returns the default value specified in the 2nd argument if the custom setting is not set in the current profile. +)", + .syntax = "getSettingOrDefault('custom_setting', default_value)", + .arguments = { + {"custom_setting", "The setting name. Type: String."}, + {"default_value", "Value to return if custom_setting is not set. Value may be of any data type or Null."}, + }, + .returned_value = "The setting's current value or the default_value if setting is not set.", + .examples = { + {"getSettingOrDefault", "SELECT getSettingOrDefault('custom_undef1', 'my_value');", "my_value"}, + {"getSettingOrDefault", "SELECT getSettingOrDefault('custom_undef1', 100);", "100"}, + {"getSettingOrDefault", "SELECT getSettingOrDefault('custom_undef1', NULL);", "NULL"}, + }, + .categories{"Other"}}, FunctionFactory::Case::Sensitive); } } diff --git a/src/Functions/materialize.cpp b/src/Functions/materialize.cpp index 5cef610b60a..e8a43dfc820 100644 --- a/src/Functions/materialize.cpp +++ b/src/Functions/materialize.cpp @@ -7,7 +7,7 @@ namespace DB REGISTER_FUNCTION(Materialize) { - factory.registerFunction(); + factory.registerFunction>(); } } diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 571391faba7..ac4a01d875e 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -9,13 +9,14 @@ namespace DB /** materialize(x) - materialize the constant */ +template class FunctionMaterialize : public IFunction { public: static constexpr auto name = "materialize"; static FunctionPtr create(ContextPtr) { - return std::make_shared(); + return std::make_shared>(); } /// Get the function name. @@ -55,7 +56,10 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - return recursiveRemoveSparse(arguments[0].column->convertToFullColumnIfConst()); + auto res = arguments[0].column->convertToFullColumnIfConst(); + if constexpr (remove_sparse) + res = recursiveRemoveSparse(res); + return res; } bool hasInformationAboutMonotonicity() const override { return true; } diff --git a/src/Functions/sqid.cpp b/src/Functions/sqid.cpp index 074a34bd083..0dfca39335d 100644 --- a/src/Functions/sqid.cpp +++ b/src/Functions/sqid.cpp @@ -123,7 +123,8 @@ public: { std::string_view sqid = col_non_const->getDataAt(i).toView(); std::vector integers = sqids.decode(String(sqid)); - res_nested_data.insert(integers.begin(), integers.end()); + if (!integers.empty()) + res_nested_data.insert(integers.begin(), integers.end()); res_offsets_data.push_back(res_offsets_data.back() + integers.size()); } } diff --git a/src/IO/AsynchronousReadBufferFromFile.cpp b/src/IO/AsynchronousReadBufferFromFile.cpp index 4d148a5c9f9..bfadb92ee77 100644 --- a/src/IO/AsynchronousReadBufferFromFile.cpp +++ b/src/IO/AsynchronousReadBufferFromFile.cpp @@ -82,7 +82,7 @@ AsynchronousReadBufferFromFile::~AsynchronousReadBufferFromFile() if (fd < 0) return; - int err = ::close(fd); + [[maybe_unused]] int err = ::close(fd); chassert(!err || errno == EINTR); } diff --git a/src/IO/ReadBufferFromFile.cpp b/src/IO/ReadBufferFromFile.cpp index 2e5b9fcf753..3d0b75c39de 100644 --- a/src/IO/ReadBufferFromFile.cpp +++ b/src/IO/ReadBufferFromFile.cpp @@ -77,7 +77,7 @@ ReadBufferFromFile::~ReadBufferFromFile() if (fd < 0) return; - int err = ::close(fd); + [[maybe_unused]] int err = ::close(fd); chassert(!err || errno == EINTR); } diff --git a/src/IO/ReadSettings.cpp b/src/IO/ReadSettings.cpp new file mode 100644 index 00000000000..70ae3cc0edf --- /dev/null +++ b/src/IO/ReadSettings.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +#include + +namespace DB +{ + +ReadSettings getReadSettings() +{ + auto query_context = CurrentThread::getQueryContext(); + if (query_context) + return query_context->getReadSettings(); + + auto global_context = Context::getGlobalContextInstance(); + if (global_context) + return global_context->getReadSettings(); + + return {}; +} + +} diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 4f7c962ed9b..7dcb08e961f 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -63,9 +63,13 @@ enum class RemoteFSReadMethod : uint8_t class MMappedFileCache; class PageCache; +class Context; struct ReadSettings { + ReadSettings() = default; + explicit ReadSettings(const Context & context); + /// Method to use reading from local filesystem. LocalFSReadMethod local_fs_method = LocalFSReadMethod::pread; /// Method to use reading from remote filesystem. @@ -139,4 +143,6 @@ struct ReadSettings } }; +ReadSettings getReadSettings(); + } diff --git a/src/IO/WriteBufferFromFile.cpp b/src/IO/WriteBufferFromFile.cpp index d68203029c1..8739d99e8d5 100644 --- a/src/IO/WriteBufferFromFile.cpp +++ b/src/IO/WriteBufferFromFile.cpp @@ -91,7 +91,7 @@ WriteBufferFromFile::~WriteBufferFromFile() tryLogCurrentException(__PRETTY_FUNCTION__); } - int err = ::close(fd); + [[maybe_unused]] int err = ::close(fd); /// Everything except for EBADF should be ignored in dtor, since all of /// others (EINTR/EIO/ENOSPC/EDQUOT) could be possible during writing to /// fd, and then write already failed and the error had been reported to diff --git a/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp index 56608644425..fe5837f31dd 100644 --- a/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateQuotaQuery.cpp @@ -111,7 +111,7 @@ BlockIO InterpreterCreateQuotaQuery::execute() if (query.alter) { - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_quota = typeid_cast>(entity->clone()); updateQuotaFromQueryImpl(*updated_quota, query, {}, roles_from_query); diff --git a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp index 4936bd15262..4c084309e37 100644 --- a/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateRoleQuery.cpp @@ -74,7 +74,7 @@ BlockIO InterpreterCreateRoleQuery::execute() if (query.alter) { - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_role = typeid_cast>(entity->clone()); updateRoleFromQueryImpl(*updated_role, query, {}, settings_from_query); diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp index b48c3880c59..8790d38b8d9 100644 --- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp @@ -88,7 +88,7 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() Strings names = query.names->toStrings(); if (query.alter) { - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_policy = typeid_cast>(entity->clone()); updateRowPolicyFromQueryImpl(*updated_policy, query, {}, roles_from_query); diff --git a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp index 029deff9b22..aadc00d3e6f 100644 --- a/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateSettingsProfileQuery.cpp @@ -90,7 +90,7 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() if (query.alter) { - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_profile = typeid_cast>(entity->clone()); updateSettingsProfileFromQueryImpl(*updated_profile, query, {}, settings_from_query, roles_from_query); diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 81600b2b6eb..adb4bc2283a 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -264,7 +264,7 @@ BlockIO InterpreterCreateUserQuery::execute() if (query.grantees) grantees_from_query = RolesOrUsersSet{*query.grantees, access_control}; - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); updateUserFromQueryImpl( @@ -317,7 +317,7 @@ BlockIO InterpreterCreateUserQuery::execute() if (query.grantees) { RolesOrUsersSet grantees_from_query = RolesOrUsersSet{*query.grantees, access_control}; - access_control.update(ids, [&](const AccessEntityPtr & entity) -> AccessEntityPtr + access_control.update(ids, [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); updated_user->grantees = grantees_from_query; diff --git a/src/Interpreters/Access/InterpreterGrantQuery.cpp b/src/Interpreters/Access/InterpreterGrantQuery.cpp index ac3b549a576..99d574dba1e 100644 --- a/src/Interpreters/Access/InterpreterGrantQuery.cpp +++ b/src/Interpreters/Access/InterpreterGrantQuery.cpp @@ -474,7 +474,7 @@ BlockIO InterpreterGrantQuery::execute() calculateCurrentGrantRightsWithIntersection(new_rights, current_user_access, elements_to_grant); /// Update roles and users listed in `grantees`. - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto clone = entity->clone(); if (query.current_grants) diff --git a/src/Interpreters/Access/InterpreterSetRoleQuery.cpp b/src/Interpreters/Access/InterpreterSetRoleQuery.cpp index 99a7a73d46c..78926c3a0c9 100644 --- a/src/Interpreters/Access/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/Access/InterpreterSetRoleQuery.cpp @@ -46,7 +46,7 @@ void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query) std::vector to_users = RolesOrUsersSet{*query.to_users, access_control, getContext()->getUserID()}.getMatchingIDs(access_control); RolesOrUsersSet roles_from_query{*query.roles, access_control}; - auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + auto update_func = [&](const AccessEntityPtr & entity, const UUID &) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); updateUserSetDefaultRoles(*updated_user, roles_from_query); diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 3bf79dc173f..e536ca9c0c6 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1433,16 +1433,21 @@ bool ActionsDAG::hasNonDeterministic() const return false; } -void ActionsDAG::addMaterializingOutputActions() +void ActionsDAG::addMaterializingOutputActions(bool materialize_sparse) { for (auto & output_node : outputs) - output_node = &materializeNode(*output_node); + output_node = &materializeNode(*output_node, materialize_sparse); } -const ActionsDAG::Node & ActionsDAG::materializeNode(const Node & node) +const ActionsDAG::Node & ActionsDAG::materializeNode(const Node & node, bool materialize_sparse) { - FunctionOverloadResolverPtr func_builder_materialize - = std::make_unique(std::make_shared()); + FunctionPtr func_materialize; + if (materialize_sparse) + func_materialize = std::make_shared>(); + else + func_materialize = std::make_shared>(); + + FunctionOverloadResolverPtr func_builder_materialize = std::make_unique(std::move(func_materialize)); const auto & name = node.result_name; const auto * func = &addFunction(func_builder_materialize, {&node}, {}); @@ -1469,7 +1474,7 @@ ActionsDAG ActionsDAG::makeConvertingActions( ActionsDAG actions_dag(source); NodeRawConstPtrs projection(num_result_columns); - FunctionOverloadResolverPtr func_builder_materialize = std::make_unique(std::make_shared()); + FunctionOverloadResolverPtr func_builder_materialize = std::make_unique(std::make_shared>()); std::unordered_map> inputs; if (mode == MatchColumnsMode::Name) @@ -1596,7 +1601,7 @@ ActionsDAG ActionsDAG::makeAddingColumnActions(ColumnWithTypeAndName column) { ActionsDAG adding_column_action; FunctionOverloadResolverPtr func_builder_materialize - = std::make_unique(std::make_shared()); + = std::make_unique(std::make_shared>()); auto column_name = column.name; const auto * column_node = &adding_column_action.addColumn(std::move(column)); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index f948e26e32e..ac92d99bfa8 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -282,14 +282,13 @@ public: /// For apply materialize() function for every output. /// Also add aliases so the result names remain unchanged. - void addMaterializingOutputActions(); + void addMaterializingOutputActions(bool materialize_sparse); /// Apply materialize() function to node. Result node has the same name. - const Node & materializeNode(const Node & node); + const Node & materializeNode(const Node & node, bool materialize_sparse = true); enum class MatchColumnsMode : uint8_t { - /// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names. Position, /// Find columns in source by their names. Allow excessive columns in source. Name, diff --git a/src/Interpreters/BloomFilterHash.h b/src/Interpreters/BloomFilterHash.h index 8248e9e4469..49450b5932b 100644 --- a/src/Interpreters/BloomFilterHash.h +++ b/src/Interpreters/BloomFilterHash.h @@ -171,7 +171,7 @@ struct BloomFilterHash const auto * index_column = typeid_cast *>(column); if (unlikely(!index_column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column type was passed to the bloom filter index."); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} was passed to the bloom filter index", column->getName()); const typename ColumnVector::Container & vec_from = index_column->getData(); diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 323527bc324..398a48c790b 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -849,7 +849,7 @@ LockedKey::~LockedKey() /// See comment near cleanupThreadFunc() for more details. key_metadata->key_state = KeyMetadata::KeyState::REMOVING; - LOG_TRACE(key_metadata->logger(), "Submitting key {} for removal", getKey()); + LOG_TEST(key_metadata->logger(), "Submitting key {} for removal", getKey()); key_metadata->addToCleanupQueue(); } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d1bc7b977c8..8629863125e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -253,13 +253,13 @@ namespace ErrorCodes extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int LOGICAL_ERROR; extern const int INVALID_SETTING_VALUE; - extern const int UNKNOWN_READ_METHOD; extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_FUNCTION; extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; extern const int CLUSTER_DOESNT_EXIST; extern const int SET_NON_GRANTED_ROLE; + extern const int UNKNOWN_READ_METHOD; } #define SHUTDOWN(log, desc, ptr, method) do \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7c448d07d93..9e3f38cc4a9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -5,17 +5,20 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -871,6 +874,43 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti } properties.constraints = as_storage_metadata->getConstraints(); + + if (create.is_clone_as) + { + if (!endsWith(as_storage->getName(), "MergeTree")) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS from tables of the MergeTree family"); + + if (create.storage) + { + if (!endsWith(create.storage->engine->name, "MergeTree")) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family"); + + /// Ensure that as_storage and the new storage has the same primary key, sorting key and partition key + auto query_to_string = [](const IAST * ast) { return ast ? queryToString(*ast) : ""; }; + + const String as_storage_sorting_key_str = query_to_string(as_storage_metadata->getSortingKeyAST().get()); + const String as_storage_primary_key_str = query_to_string(as_storage_metadata->getPrimaryKeyAST().get()); + const String as_storage_partition_key_str = query_to_string(as_storage_metadata->getPartitionKeyAST().get()); + + const String storage_sorting_key_str = query_to_string(create.storage->order_by); + const String storage_primary_key_str = query_to_string(create.storage->primary_key); + const String storage_partition_key_str = query_to_string(create.storage->partition_by); + + if (as_storage_sorting_key_str != storage_sorting_key_str) + { + /// It is possible that the storage only has primary key and an empty sorting key, and as_storage has both primary key and sorting key with the same value. + if (as_storage_sorting_key_str != as_storage_primary_key_str || as_storage_sorting_key_str != storage_primary_key_str) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different ordering"); + } + } + if (as_storage_partition_key_str != storage_partition_key_str) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key"); + + if (as_storage_primary_key_str != storage_primary_key_str) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different primary key"); + } + } } else if (create.select) { @@ -1537,37 +1577,47 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.select && create.is_materialized_view && mode <= LoadingStrictnessLevel::CREATE) validateMaterializedViewColumnsAndEngine(create, properties, database); - bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; - if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + bool is_storage_replicated = false; + if (create.storage && isReplicated(*create.storage)) + is_storage_replicated = true; + + if (create.targets) { - bool is_storage_replicated = false; - - if (create.storage && isReplicated(*create.storage)) - is_storage_replicated = true; - - if (create.targets) + for (const auto & inner_table_engine : create.targets->getInnerEngines()) { - for (const auto & inner_table_engine : create.targets->getInnerEngines()) + if (isReplicated(*inner_table_engine)) + is_storage_replicated = true; + } + } + + bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; + if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + { + const bool allow_create_select_for_replicated + = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; + if (!allow_create_select_for_replicated) { - if (isReplicated(*inner_table_engine)) - is_storage_replicated = true; + /// POPULATE can be enabled with setting, provide hint in error message + if (create.is_populate) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " + "queries. " + "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " + "caution"); + + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); } } - const bool allow_create_select_for_replicated = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; - if (!allow_create_select_for_replicated) - { - /// POPULATE can be enabled with setting, provide hint in error message - if (create.is_populate) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT queries. " - "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with caution"); - + if (create.is_clone_as) + { + if (database && database->getEngineName() == "Replicated") throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - } + "CREATE CLONE AS is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); } if (database && database->shouldReplicateQuery(getContext(), query_ptr)) @@ -1588,7 +1638,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (need_add_to_database && !database) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name)); - if (create.replace_table) + if (create.replace_table + || (create.replace_view && (database->getEngineName() == "Atomic" || database->getEngineName() == "Replicated"))) { chassert(!ddl_guard); return doCreateOrReplaceTable(create, properties, mode); @@ -1944,16 +1995,16 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, auto ast_rename = std::make_shared(ASTRenameQuery::Elements{std::move(elem)}); ast_rename->dictionary = create.is_dictionary; - if (create.create_or_replace) + if (create.create_or_replace || create.replace_view) { - /// CREATE OR REPLACE TABLE + /// CREATE OR REPLACE TABLE/VIEW /// Will execute ordinary RENAME instead of EXCHANGE if the target table does not exist ast_rename->rename_if_cannot_exchange = true; ast_rename->exchange = false; } else { - /// REPLACE TABLE + /// REPLACE TABLE/VIEW /// Will execute EXCHANGE query and fail if the target table does not exist ast_rename->exchange = true; } @@ -2012,6 +2063,38 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) .execute(); } + /// If the query is a CREATE TABLE .. CLONE AS ..., attach all partitions of the source table to the newly created table. + if (create.is_clone_as && !as_table_saved.empty() && !create.is_create_empty && !create.is_ordinary_view && !create.is_live_view + && (!(create.is_materialized_view || create.is_window_view) || create.is_populate)) + { + String as_database_name = getContext()->resolveDatabase(create.as_database); + + auto partition = std::make_shared(); + partition->all = true; + + auto command = std::make_shared(); + command->replace = false; + command->type = ASTAlterCommand::REPLACE_PARTITION; + command->partition = command->children.emplace_back(std::move(partition)).get(); + command->from_database = as_database_name; + command->from_table = as_table_saved; + command->to_database = create.getDatabase(); + command->to_table = create.getTable(); + + auto command_list = std::make_shared(); + command_list->children.push_back(command); + + auto query = std::make_shared(); + query->database = create.database; + query->table = create.table; + query->uuid = create.uuid; + auto * alter = query->as(); + + alter->alter_object = ASTAlterQuery::AlterObjectType::TABLE; + alter->set(alter->command_list, command_list); + return InterpreterAlterQuery(query, getContext()).execute(); + } + return {}; } diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index d7f9c338ab1..ef7d90d3ac1 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -71,6 +71,7 @@ namespace Setting extern const SettingsBool use_concurrency_control; extern const SettingsSeconds lock_acquire_timeout; extern const SettingsUInt64 parallel_distributed_insert_select; + extern const SettingsBool enable_parsing_to_custom_serialization; } namespace ErrorCodes @@ -217,7 +218,7 @@ Block InterpreterInsertQuery::getSampleBlockImpl( for (const auto & current_name : names) { if (res.has(current_name)) - throw Exception(ErrorCodes::DUPLICATE_COLUMN, "Column {} specified more than once", current_name); + throw Exception(ErrorCodes::DUPLICATE_COLUMN, "Column {} in table {} specified more than once", current_name, table->getStorageID().getNameForLogs()); /// Column is not ordinary or ephemeral if (!table_sample_insertable.has(current_name)) @@ -563,11 +564,10 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & return std::make_shared(in_header, actions); }); - /// We need to convert Sparse columns to full, because it's destination storage - /// may not support it or may have different settings for applying Sparse serialization. + /// We need to convert Sparse columns to full if the destination storage doesn't support them. pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr { - return std::make_shared(in_header); + return std::make_shared(in_header, !table->supportsSparseSerialization()); }); pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr @@ -737,11 +737,14 @@ QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query if (query.hasInlinedData() && !async_insert) { - /// can execute without additional data auto format = getInputFormatFromASTInsertQuery(query_ptr, true, query_sample_block, getContext(), nullptr); - for (auto && buffer : owned_buffers) + + for (auto & buffer : owned_buffers) format->addBuffer(std::move(buffer)); + if (settings[Setting::enable_parsing_to_custom_serialization]) + format->setSerializationHints(table->getSerializationHints()); + auto pipe = getSourceFromInputFormat(query_ptr, std::move(format), getContext(), nullptr); pipeline.complete(std::move(pipe)); } diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp index c656a1a797b..8122800f882 100644 --- a/src/Interpreters/Squashing.cpp +++ b/src/Interpreters/Squashing.cpp @@ -1,8 +1,9 @@ #include #include -#include "Common/Logger.h" -#include "Common/logger_useful.h" #include +#include +#include +#include #include namespace DB @@ -116,7 +117,7 @@ Chunk Squashing::squash(std::vector && input_chunks, Chunk::ChunkInfoColl return result; } - std::vector mutable_columns = {}; + std::vector mutable_columns; size_t rows = 0; for (const Chunk & chunk : input_chunks) rows += chunk.getNumRows(); @@ -130,8 +131,11 @@ Chunk Squashing::squash(std::vector && input_chunks, Chunk::ChunkInfoColl } size_t num_columns = mutable_columns.size(); + /// Collect the list of source columns for each column. - std::vector source_columns_list(num_columns, Columns{}); + std::vector source_columns_list(num_columns); + std::vector have_same_serialization(num_columns, true); + for (size_t i = 0; i != num_columns; ++i) source_columns_list[i].reserve(input_chunks.size() - 1); @@ -139,11 +143,21 @@ Chunk Squashing::squash(std::vector && input_chunks, Chunk::ChunkInfoColl { auto columns = input_chunks[i].detachColumns(); for (size_t j = 0; j != num_columns; ++j) + { + have_same_serialization[j] &= ISerialization::getKind(*columns[j]) == ISerialization::getKind(*mutable_columns[j]); source_columns_list[j].emplace_back(std::move(columns[j])); + } } for (size_t i = 0; i != num_columns; ++i) { + if (!have_same_serialization[i]) + { + mutable_columns[i] = recursiveRemoveSparse(std::move(mutable_columns[i]))->assumeMutable(); + for (auto & column : source_columns_list[i]) + column = recursiveRemoveSparse(column); + } + /// We know all the data we will insert in advance and can make all necessary pre-allocations. mutable_columns[i]->prepareForSquashing(source_columns_list[i]); for (auto & source_column : source_columns_list[i]) diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index 769757a5fba..c73bee827a4 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -134,7 +134,7 @@ ColumnsDescription ZooKeeperLogElement::getColumnsDescription() {"session_id", std::make_shared(), "The session ID that the ZooKeeper server sets for each connection."}, {"duration_microseconds", std::make_shared(), "The time taken by ZooKeeper to execute the request."}, - {"xid", std::make_shared(), "The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired response/finalize row."}, + {"xid", std::make_shared(), "The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired response/finalize row."}, {"has_watch", std::make_shared(), "The request whether the watch has been set."}, {"op_num", op_num_enum, "The type of request or response."}, {"path", std::make_shared(), "The path to the ZooKeeper node specified in the request, or an empty string if the request not requires specifying a path."}, diff --git a/src/Interpreters/ZooKeeperLog.h b/src/Interpreters/ZooKeeperLog.h index d3868c01202..88eabc85564 100644 --- a/src/Interpreters/ZooKeeperLog.h +++ b/src/Interpreters/ZooKeeperLog.h @@ -31,7 +31,7 @@ struct ZooKeeperLogElement UInt64 duration_microseconds = 0; /// Common request info - Int32 xid = 0; + Int64 xid = 0; bool has_watch = false; Int32 op_num = 0; String path; diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index 27d79e86622..173478332f3 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -85,7 +85,7 @@ ActionsDAG addMissingDefaults( /// Removes unused columns and reorders result. actions.removeUnusedActions(required_columns.getNames(), false); - actions.addMaterializingOutputActions(); + actions.addMaterializingOutputActions(/*materialize_sparse=*/ false); return actions; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index a96350c7ca3..066de0c7d76 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -847,7 +847,13 @@ static std::tuple executeQueryImpl( /// Verify that AST formatting is consistent: /// If you format AST, parse it back, and format it again, you get the same string. - String formatted1 = ast->formatWithPossiblyHidingSensitiveData(0, true, true, false, false, IdentifierQuotingStyle::Backticks); + String formatted1 = ast->formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/true, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); /// The query can become more verbose after formatting, so: size_t new_max_query_size = max_query_size > 0 ? (1000 + 2 * max_query_size) : 0; @@ -876,7 +882,13 @@ static std::tuple executeQueryImpl( chassert(ast2); - String formatted2 = ast2->formatWithPossiblyHidingSensitiveData(0, true, true, false, false, IdentifierQuotingStyle::Backticks); + String formatted2 = ast2->formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/true, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); if (formatted1 != formatted2) throw Exception(ErrorCodes::LOGICAL_ERROR, @@ -1247,7 +1259,6 @@ static std::tuple executeQueryImpl( { if (!interpreter->supportsTransactions()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Transactions are not supported for this type of query ({})", ast->getID()); - } // InterpreterSelectQueryAnalyzer does not build QueryPlan in the constructor. diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.cpp b/src/Interpreters/formatWithPossiblyHidingSecrets.cpp index ba02a9f164b..9b7a58332a2 100644 --- a/src/Interpreters/formatWithPossiblyHidingSecrets.cpp +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.cpp @@ -8,8 +8,8 @@ namespace DB namespace Setting { extern const SettingsBool format_display_secrets_in_show_and_select; - extern const SettingsBool output_format_always_quote_identifiers; - extern const SettingsIdentifierQuotingStyle output_format_identifier_quoting_style; + extern const SettingsIdentifierQuotingRule show_create_query_identifier_quoting_rule; + extern const SettingsIdentifierQuotingStyle show_create_query_identifier_quoting_style; extern const SettingsBool print_pretty_type_names; } @@ -24,8 +24,8 @@ String format(const SecretHidingFormatSettings & settings) settings.one_line, show_secrets, settings.ctx->getSettingsRef()[Setting::print_pretty_type_names], - settings.ctx->getSettingsRef()[Setting::output_format_always_quote_identifiers], - settings.ctx->getSettingsRef()[Setting::output_format_identifier_quoting_style]); + settings.ctx->getSettingsRef()[Setting::show_create_query_identifier_quoting_rule], + settings.ctx->getSettingsRef()[Setting::show_create_query_identifier_quoting_style]); } } diff --git a/src/Parsers/ASTAssignment.h b/src/Parsers/ASTAssignment.h index a37a31ae38e..2b1391b5812 100644 --- a/src/Parsers/ASTAssignment.h +++ b/src/Parsers/ASTAssignment.h @@ -30,7 +30,7 @@ protected: { settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(column_name); + settings.writeIdentifier(column_name, /*ambiguous=*/false); settings.ostr << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_operator : "") << " = " << (settings.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index d7728462df3..e7c3fdbb548 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -66,8 +66,7 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & format_settings, Fo { frame.need_parens = false; - /// We have to always quote column names to avoid ambiguity with INDEX and other declarations in CREATE query. - format_settings.quoteIdentifier(name); + format_settings.writeIdentifier(name, /*ambiguous=*/true); if (type) { diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index d7f5b8f9702..66422efe660 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -425,9 +425,19 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : ""); }; + bool should_add_clone = is_clone_as; + auto add_clone_if_needed = [&] + { + if (!should_add_clone) + return; + should_add_clone = false; + settings.ostr << (settings.hilite ? hilite_keyword : "") << " CLONE" << (settings.hilite ? hilite_none : ""); + }; + if (!as_table.empty()) { add_empty_if_needed(); + add_clone_if_needed(); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table); @@ -446,6 +456,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat } add_empty_if_needed(); + add_clone_if_needed(); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); as_table_function->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 6be0fa78903..813e56eaf02 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -100,6 +100,7 @@ public: bool is_time_series_table{false}; /// CREATE TABLE ... ENGINE=TimeSeries() ... bool is_populate{false}; bool is_create_empty{false}; /// CREATE TABLE ... EMPTY AS SELECT ... + bool is_clone_as{false}; /// CREATE TABLE ... CLONE AS ... bool replace_view{false}; /// CREATE OR REPLACE VIEW bool has_uuid{false}; // CREATE TABLE x UUID '...' diff --git a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp index a600987dc45..43fef42be4f 100644 --- a/src/Parsers/ASTDictionaryAttributeDeclaration.cpp +++ b/src/Parsers/ASTDictionaryAttributeDeclaration.cpp @@ -35,7 +35,7 @@ void ASTDictionaryAttributeDeclaration::formatImpl(const FormatSettings & settin { frame.need_parens = false; - settings.quoteIdentifier(name); + settings.writeIdentifier(name, /*ambiguous=*/true); if (type) { diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 80a618170c6..f466e10d7d8 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -108,7 +108,7 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form auto format_element = [&](const String & elem_name) { settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(elem_name); + settings.writeIdentifier(elem_name, /*ambiguous=*/false); settings.ostr << (settings.hilite ? hilite_none : ""); }; diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index 1e39eacaac7..80102479cb9 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -79,7 +79,7 @@ void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & sta } else { - s.writeIdentifier(name); + s.writeIdentifier(name, /*ambiguous=*/false); s.ostr << " "; expr->formatImpl(s, state, frame); } diff --git a/src/Parsers/ASTProjectionDeclaration.cpp b/src/Parsers/ASTProjectionDeclaration.cpp index af79745a88e..21a924c0aff 100644 --- a/src/Parsers/ASTProjectionDeclaration.cpp +++ b/src/Parsers/ASTProjectionDeclaration.cpp @@ -17,7 +17,7 @@ ASTPtr ASTProjectionDeclaration::clone() const void ASTProjectionDeclaration::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.writeIdentifier(name); + settings.writeIdentifier(name, /*ambiguous=*/false); std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); std::string nl_or_nothing = settings.one_line ? "" : "\n"; settings.ostr << settings.nl_or_ws << indent_str << "(" << nl_or_nothing; diff --git a/src/Parsers/ASTSubquery.cpp b/src/Parsers/ASTSubquery.cpp index 844520b2f64..7b4f8d141fd 100644 --- a/src/Parsers/ASTSubquery.cpp +++ b/src/Parsers/ASTSubquery.cpp @@ -35,7 +35,7 @@ void ASTSubquery::formatImplWithoutAlias(const FormatSettings & settings, Format if (!cte_name.empty()) { settings.ostr << (settings.hilite ? hilite_identifier : ""); - settings.writeIdentifier(cte_name); + settings.writeIdentifier(cte_name, /*ambiguous=*/false); settings.ostr << (settings.hilite ? hilite_none : ""); return; } diff --git a/src/Parsers/ASTWithAlias.cpp b/src/Parsers/ASTWithAlias.cpp index 6f64e33d33d..a5c285f315e 100644 --- a/src/Parsers/ASTWithAlias.cpp +++ b/src/Parsers/ASTWithAlias.cpp @@ -10,7 +10,7 @@ namespace DB static void writeAlias(const String & name, const ASTWithAlias::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " AS " << (settings.hilite ? IAST::hilite_alias : ""); - settings.writeIdentifier(name); + settings.writeIdentifier(name, /*ambiguous=*/false); settings.ostr << (settings.hilite ? IAST::hilite_none : ""); } @@ -22,7 +22,7 @@ void ASTWithAlias::formatImpl(const FormatSettings & settings, FormatState & sta if (!alias.empty() && !state.printed_asts_with_alias.emplace(frame.current_select, alias, getTreeHash(/*ignore_aliases=*/ true)).second) { settings.ostr << (settings.hilite ? IAST::hilite_identifier : ""); - settings.writeIdentifier(alias); + settings.writeIdentifier(alias, /*ambiguous=*/false); settings.ostr << (settings.hilite ? IAST::hilite_none : ""); } else diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp index c2cb1177c17..a94b0a7062d 100644 --- a/src/Parsers/ASTWithElement.cpp +++ b/src/Parsers/ASTWithElement.cpp @@ -19,7 +19,7 @@ void ASTWithElement::formatImpl(const FormatSettings & settings, FormatState & s std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); settings.ostr << (settings.hilite ? hilite_alias : ""); - settings.writeIdentifier(name); + settings.writeIdentifier(name, /*ambiguous=*/false); settings.ostr << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS" << (settings.hilite ? hilite_none : ""); settings.ostr << settings.nl_or_ws << indent_str; diff --git a/src/Parsers/CommonParsers.cpp b/src/Parsers/CommonParsers.cpp index ed8d58d5a68..f07a0e8d319 100644 --- a/src/Parsers/CommonParsers.cpp +++ b/src/Parsers/CommonParsers.cpp @@ -1,9 +1,8 @@ -#include -#include -#include #include #include - +#include +#include +#include namespace DB { @@ -35,6 +34,8 @@ public: return mapping; } + const std::unordered_set & getSet() const { return set; } + private: KeyWordToStringConverter() { @@ -63,6 +64,7 @@ private: size_t index = static_cast(identifier); mapping.resize(std::max(index + 1, mapping.size())); mapping[index] = value; + set.emplace(value); } void checkUnderscore(std::string_view value) @@ -94,6 +96,7 @@ private: } Strings mapping; + std::unordered_set set; }; } @@ -108,6 +111,11 @@ const std::vector & getAllKeyWords() return KeyWordToStringConverter::instance().getMapping(); } +const std::unordered_set & getKeyWordSet() +{ + return KeyWordToStringConverter::instance().getSet(); +} + ParserKeyword::ParserKeyword(Keyword keyword) : s(toStringView(keyword)) {} diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 46e08cf3f7e..00be5aa6a05 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -84,6 +85,7 @@ namespace DB MR_MACROS(CLEAR_INDEX, "CLEAR INDEX") \ MR_MACROS(CLEAR_PROJECTION, "CLEAR PROJECTION") \ MR_MACROS(CLEAR_STATISTICS, "CLEAR STATISTICS") \ + MR_MACROS(CLONE_AS, "CLONE AS") \ MR_MACROS(CLUSTER, "CLUSTER") \ MR_MACROS(CLUSTERS, "CLUSTERS") \ MR_MACROS(CN, "CN") \ @@ -588,6 +590,8 @@ std::string_view toStringView(Keyword type); const std::vector & getAllKeyWords(); +const std::unordered_set & getKeyWordSet(); + /** Parse specified keyword such as SELECT or compound keyword such as ORDER BY. * All case insensitive. Requires word boundary. diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 54b11f2888e..d6daf9bd78b 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -1,12 +1,14 @@ #include +#include #include #include -#include +#include +#include +#include #include #include - namespace DB { @@ -14,7 +16,6 @@ namespace ErrorCodes { extern const int TOO_BIG_AST; extern const int TOO_DEEP_AST; - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_ELEMENT_IN_AST; } @@ -170,7 +171,7 @@ String IAST::formatWithPossiblyHidingSensitiveData( bool one_line, bool show_secrets, bool print_pretty_type_names, - bool always_quote_identifiers, + IdentifierQuotingRule identifier_quoting_rule, IdentifierQuotingStyle identifier_quoting_style) const { @@ -178,7 +179,7 @@ String IAST::formatWithPossiblyHidingSensitiveData( FormatSettings settings(buf, one_line); settings.show_secrets = show_secrets; settings.print_pretty_type_names = print_pretty_type_names; - settings.always_quote_identifiers = always_quote_identifiers; + settings.identifier_quoting_rule = identifier_quoting_rule; settings.identifier_quoting_style = identifier_quoting_style; format(settings); return wipeSensitiveDataAndCutToLength(buf.str(), max_length); @@ -217,22 +218,24 @@ String IAST::getColumnNameWithoutAlias() const } -void IAST::FormatSettings::writeIdentifier(const String & name) const +void IAST::FormatSettings::writeIdentifier(const String & name, bool ambiguous) const { + bool must_quote + = (identifier_quoting_rule == IdentifierQuotingRule::Always + || (ambiguous && identifier_quoting_rule == IdentifierQuotingRule::WhenNecessary)); + + if (identifier_quoting_rule == IdentifierQuotingRule::UserDisplay && !must_quote) + { + // Quote `name` if it is one of the keywords when `identifier_quoting_rule` is `IdentifierQuotingRule::UserDisplay` + const auto & keyword_set = getKeyWordSet(); + must_quote = keyword_set.contains(Poco::toUpper(name)); + } + switch (identifier_quoting_style) { - case IdentifierQuotingStyle::None: - { - if (always_quote_identifiers) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Incompatible arguments: always_quote_identifiers = true && " - "identifier_quoting_style == IdentifierQuotingStyle::None"); - writeString(name, ostr); - break; - } case IdentifierQuotingStyle::Backticks: { - if (always_quote_identifiers) + if (must_quote) writeBackQuotedString(name, ostr); else writeProbablyBackQuotedString(name, ostr); @@ -240,7 +243,7 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const } case IdentifierQuotingStyle::DoubleQuotes: { - if (always_quote_identifiers) + if (must_quote) writeDoubleQuotedString(name, ostr); else writeProbablyDoubleQuotedString(name, ostr); @@ -248,7 +251,7 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const } case IdentifierQuotingStyle::BackticksMySQL: { - if (always_quote_identifiers) + if (must_quote) writeBackQuotedStringMySQL(name, ostr); else writeProbablyBackQuotedStringMySQL(name, ostr); @@ -257,34 +260,6 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const } } - -void IAST::FormatSettings::quoteIdentifier(const String & name) const -{ - switch (identifier_quoting_style) - { - case IdentifierQuotingStyle::None: - { - writeBackQuotedString(name, ostr); - break; - } - case IdentifierQuotingStyle::Backticks: - { - writeBackQuotedString(name, ostr); - break; - } - case IdentifierQuotingStyle::DoubleQuotes: - { - writeDoubleQuotedString(name, ostr); - break; - } - case IdentifierQuotingStyle::BackticksMySQL: - { - writeBackQuotedStringMySQL(name, ostr); - break; - } - } -} - void IAST::dumpTree(WriteBuffer & ostr, size_t indent) const { String indent_str(indent, '-'); diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index dfb6a6cbeba..8bf20c40b70 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -196,7 +196,7 @@ public: WriteBuffer & ostr; bool one_line; bool hilite; - bool always_quote_identifiers; + IdentifierQuotingRule identifier_quoting_rule; IdentifierQuotingStyle identifier_quoting_style; bool show_secrets; /// Show secret parts of the AST (e.g. passwords, encryption keys). char nl_or_ws; /// Newline or whitespace. @@ -207,7 +207,7 @@ public: WriteBuffer & ostr_, bool one_line_, bool hilite_ = false, - bool always_quote_identifiers_ = false, + IdentifierQuotingRule identifier_quoting_rule_ = IdentifierQuotingRule::WhenNecessary, IdentifierQuotingStyle identifier_quoting_style_ = IdentifierQuotingStyle::Backticks, bool show_secrets_ = true, LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular, @@ -215,7 +215,7 @@ public: : ostr(ostr_) , one_line(one_line_) , hilite(hilite_) - , always_quote_identifiers(always_quote_identifiers_) + , identifier_quoting_rule(identifier_quoting_rule_) , identifier_quoting_style(identifier_quoting_style_) , show_secrets(show_secrets_) , nl_or_ws(one_line ? ' ' : '\n') @@ -228,7 +228,7 @@ public: : ostr(ostr_) , one_line(other.one_line) , hilite(other.hilite) - , always_quote_identifiers(other.always_quote_identifiers) + , identifier_quoting_rule(other.identifier_quoting_rule) , identifier_quoting_style(other.identifier_quoting_style) , show_secrets(other.show_secrets) , nl_or_ws(other.nl_or_ws) @@ -237,10 +237,7 @@ public: { } - void writeIdentifier(const String & name) const; - // Quote identifier `name` even when `always_quote_identifiers` is false. - // If `identifier_quoting_style` is `IdentifierQuotingStyle::None`, quote it with `IdentifierQuotingStyle::Backticks` - void quoteIdentifier(const String & name) const; + void writeIdentifier(const String & name, bool ambiguous) const; }; /// State. For example, a set of nodes can be remembered, which we already walk through. @@ -286,7 +283,7 @@ public: bool one_line, bool show_secrets, bool print_pretty_type_names, - bool always_quote_identifiers, + IdentifierQuotingRule identifier_quoting_rule, IdentifierQuotingStyle identifier_quoting_style) const; /** formatForLogging and formatForErrorMessage always hide secrets. This inconsistent @@ -296,12 +293,24 @@ public: */ String formatForLogging(size_t max_length = 0) const { - return formatWithPossiblyHidingSensitiveData(max_length, true, false, false, false, IdentifierQuotingStyle::Backticks); + return formatWithPossiblyHidingSensitiveData( + /*max_length=*/max_length, + /*one_line=*/true, + /*show_secrets=*/false, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); } String formatForErrorMessage() const { - return formatWithPossiblyHidingSensitiveData(0, true, false, false, false, IdentifierQuotingStyle::Backticks); + return formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/false, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); } virtual bool hasSecretParts() const { return childrenHaveSecretParts(); } diff --git a/src/Parsers/IdentifierQuotingStyle.h b/src/Parsers/IdentifierQuotingStyle.h index 48be809fc8f..b62c9fbe97d 100644 --- a/src/Parsers/IdentifierQuotingStyle.h +++ b/src/Parsers/IdentifierQuotingStyle.h @@ -8,10 +8,19 @@ namespace DB /// NOTE There could be differences in escaping rules inside quotes. Escaping rules may not match that required by specific external DBMS. enum class IdentifierQuotingStyle : uint8_t { - None, /// Write as-is, without quotes. - Backticks, /// `clickhouse` style - DoubleQuotes, /// "postgres" style - BackticksMySQL, /// `mysql` style, most same as Backticks, but it uses '``' to escape '`' + Backticks, /// `clickhouse` style + DoubleQuotes, /// "postgres" style + BackticksMySQL, /// `mysql` style, most same as Backticks, but it uses '``' to escape '`' }; +enum class IdentifierQuotingRule : uint8_t +{ + /// When the identifiers is one of {"distinct", "all", "table"} (defined in `DB::writeProbablyQuotedStringImpl`), + /// or it can cause ambiguity: column names, dictionary attribute names (passed to `DB::FormatSettings::writeIdentifier` with `ambiguous=true`) + WhenNecessary, + /// Always quote identifiers + Always, + /// When the identifiers is a keyword (defined in `DB::Keyword`) + UserDisplay, +}; } diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 31dc2075db4..1faaaaafb3a 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -68,7 +68,7 @@ bool ParserSQLSecurity::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) while (true) { - if (!definer && s_definer.ignore(pos, expected)) + if (!definer && !is_definer_current_user && s_definer.ignore(pos, expected)) { s_eq.ignore(pos, expected); if (s_current_user.ignore(pos, expected)) @@ -675,6 +675,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe bool if_not_exists = false; bool is_temporary = false; bool is_create_empty = false; + bool is_clone_as = false; if (s_create.ignore(pos, expected)) { @@ -759,13 +760,18 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; }; - auto need_parse_as_select = [&is_create_empty, &pos, &expected]() + auto need_parse_as_select = [&is_create_empty, &is_clone_as, &pos, &expected]() { if (ParserKeyword{Keyword::EMPTY_AS}.ignore(pos, expected)) { is_create_empty = true; return true; } + if (ParserKeyword{Keyword::CLONE_AS}.ignore(pos, expected)) + { + is_clone_as = true; + return true; + } return ParserKeyword{Keyword::AS}.ignore(pos, expected); }; @@ -893,6 +899,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe query->set(query->select, select); query->set(query->targets, targets); query->is_create_empty = is_create_empty; + query->is_clone_as = is_clone_as; if (from_path) query->attach_from_path = from_path->as().value.safeGet(); diff --git a/src/Parsers/fuzzers/codegen_fuzzer/gen.py b/src/Parsers/fuzzers/codegen_fuzzer/gen.py index 84ee09916c4..3fe65069477 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/gen.py +++ b/src/Parsers/fuzzers/codegen_fuzzer/gen.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -import sys import string - +import sys TOKEN_TEXT = 1 TOKEN_VAR = 2 diff --git a/src/Parsers/getInsertQuery.cpp b/src/Parsers/getInsertQuery.cpp index 9d111b147bd..398d4fc6b5d 100644 --- a/src/Parsers/getInsertQuery.cpp +++ b/src/Parsers/getInsertQuery.cpp @@ -19,7 +19,12 @@ std::string getInsertQuery(const std::string & db_name, const std::string & tabl query.columns->children.emplace_back(std::make_shared(column.name)); WriteBufferFromOwnString buf; - IAST::FormatSettings settings(buf, /*one_line*/ true, /*hilite*/ false, /*always_quote_identifiers*/ true, /*identifier_quoting_style*/ quoting); + IAST::FormatSettings settings( + /*ostr_=*/buf, + /*one_line=*/true, + /*hilite=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/quoting); query.IAST::format(settings); return buf.str(); } diff --git a/src/Processors/Executors/ExecutingGraph.cpp b/src/Processors/Executors/ExecutingGraph.cpp index f2927d4145c..8cf73c49f59 100644 --- a/src/Processors/Executors/ExecutingGraph.cpp +++ b/src/Processors/Executors/ExecutingGraph.cpp @@ -181,7 +181,7 @@ ExecutingGraph::UpdateNodeStatus ExecutingGraph::expandPipeline(std::stack stack; @@ -197,18 +197,12 @@ void ExecutingGraph::initializeExecution(Queue & queue) } } - Queue async_queue; - while (!stack.empty()) { uint64_t proc = stack.top(); stack.pop(); updateNode(proc, queue, async_queue); - - if (!async_queue.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Async is only possible after work() call. Processor {}", - async_queue.front()->processor->getName()); } } diff --git a/src/Processors/Executors/ExecutingGraph.h b/src/Processors/Executors/ExecutingGraph.h index 8d8ba722b3e..676e13fcc96 100644 --- a/src/Processors/Executors/ExecutingGraph.h +++ b/src/Processors/Executors/ExecutingGraph.h @@ -136,7 +136,7 @@ public: const Processors & getProcessors() const { return *processors; } /// Traverse graph the first time to update all the childless nodes. - void initializeExecution(Queue & queue); + void initializeExecution(Queue & queue, Queue & async_queue); enum class UpdateNodeStatus { diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp index d045f59a2e2..abc450777c7 100644 --- a/src/Processors/Executors/ExecutorTasks.cpp +++ b/src/Processors/Executors/ExecutorTasks.cpp @@ -170,11 +170,27 @@ void ExecutorTasks::init(size_t num_threads_, size_t use_threads_, bool profile_ } } -void ExecutorTasks::fill(Queue & queue) +void ExecutorTasks::fill(Queue & queue, [[maybe_unused]] Queue & async_queue) { std::lock_guard lock(mutex); size_t next_thread = 0; +#if defined(OS_LINUX) + while (!async_queue.empty()) + { + int fd = async_queue.front()->processor->schedule(); + async_task_queue.addTask(next_thread, async_queue.front(), fd); + async_queue.pop(); + + ++next_thread; + + /// It is important to keep queues empty for threads that are not started yet. + /// Otherwise that thread can be selected by `tryWakeUpAnyOtherThreadWithTasks()`, leading to deadlock. + if (next_thread >= use_threads) + next_thread = 0; + } +#endif + while (!queue.empty()) { task_queue.push(queue.front(), next_thread); diff --git a/src/Processors/Executors/ExecutorTasks.h b/src/Processors/Executors/ExecutorTasks.h index b2201873edf..e551ec19161 100644 --- a/src/Processors/Executors/ExecutorTasks.h +++ b/src/Processors/Executors/ExecutorTasks.h @@ -62,7 +62,7 @@ public: void pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); void init(size_t num_threads_, size_t use_threads_, bool profile_processors, bool trace_processors, ReadProgressCallback * callback); - void fill(Queue & queue); + void fill(Queue & queue, Queue & async_queue); void upscale(size_t use_threads_); void processAsyncTasks(); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index f6fab4552e8..d02599c4d17 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -355,10 +355,11 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ use_threads = cpu_slots->grantedCount(); Queue queue; - graph->initializeExecution(queue); + Queue async_queue; + graph->initializeExecution(queue, async_queue); tasks.init(num_threads, use_threads, profile_processors, trace_processors, read_progress_callback.get()); - tasks.fill(queue); + tasks.fill(queue, async_queue); if (num_threads > 1) pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, CurrentMetrics::QueryPipelineExecutorThreadsScheduled, num_threads); diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index 713c1089d28..64b289170d2 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -58,6 +58,10 @@ public: /// parallel parsing before creating this parser. virtual void setRowsReadBefore(size_t /*rows*/) {} + /// Sets the serialization hints for the columns. It allows to create columns + /// in custom serializations (e.g. Sparse) for parsing and avoid extra conversion. + virtual void setSerializationHints(const SerializationInfoByName & /*hints*/) {} + void addBuffer(std::unique_ptr buffer) { owned_buffers.emplace_back(std::move(buffer)); } void setErrorsLogger(const InputFormatErrorsLoggerPtr & errors_logger_) { errors_logger = errors_logger_; } diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 0b6c81923db..b8e8822e648 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -103,7 +103,10 @@ Chunk IRowInputFormat::read() const Block & header = getPort().getHeader(); size_t num_columns = header.columns(); - MutableColumns columns = header.cloneEmptyColumns(); + MutableColumns columns(num_columns); + + for (size_t i = 0; i < num_columns; ++i) + columns[i] = header.getByPosition(i).type->createColumn(*serializations[i]); block_missing_values.clear(); @@ -266,5 +269,10 @@ size_t IRowInputFormat::countRows(size_t) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method countRows is not implemented for input format {}", getName()); } +void IRowInputFormat::setSerializationHints(const SerializationInfoByName & hints) +{ + serializations = getPort().getHeader().getSerializations(hints); +} + } diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index f8796df8604..c6786f45ecb 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -5,6 +5,7 @@ #include #include #include +#include class Stopwatch; @@ -84,6 +85,7 @@ protected: size_t getApproxBytesReadForChunk() const override { return approx_bytes_read_for_chunk; } void setRowsReadBefore(size_t rows) override { total_rows = rows; } + void setSerializationHints(const SerializationInfoByName & hints) override; Serializations serializations; diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 79dd6e1d35b..19547317db8 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -634,6 +634,8 @@ static void buildORCSearchArgumentImpl( } /// There is no optimization with space-filling curves for ORC. case KeyCondition::RPNElement::FUNCTION_ARGS_IN_HYPERRECTANGLE: + /// There is no optimization with pointInPolygon for ORC. + case KeyCondition::RPNElement::FUNCTION_POINT_IN_POLYGON: case KeyCondition::RPNElement::FUNCTION_UNKNOWN: { builder.literal(orc::TruthValue::YES_NO_NULL); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 447adb1ed48..faf6bf81869 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -92,6 +92,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupPtr thread_grou InputFormatPtr input_format = internal_parser_creator(read_buffer); input_format->setRowsReadBefore(unit.offset); input_format->setErrorsLogger(errors_logger); + input_format->setSerializationHints(serialization_hints); InternalParser parser(input_format); unit.chunk_ext.chunk.clear(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index b97bf5213e6..e3753385ae8 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -129,6 +129,11 @@ public: return last_block_missing_values; } + void setSerializationHints(const SerializationInfoByName & hints) override + { + serialization_hints = hints; + } + size_t getApproxBytesReadForChunk() const override { return last_approx_bytes_read_for_chunk; } String getName() const final { return "ParallelParsingBlockInputFormat"; } @@ -207,6 +212,7 @@ private: BlockMissingValues last_block_missing_values; size_t last_approx_bytes_read_for_chunk = 0; + SerializationInfoByName serialization_hints; /// Non-atomic because it is used in one thread. std::optional next_block_in_current_unit; diff --git a/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp b/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp index 46980ceb56b..2bbeba3de96 100644 --- a/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp +++ b/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp @@ -47,7 +47,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s { throw std::runtime_error("unable write to temp file"); } - int error = close(fd); + [[maybe_unused]] int error = close(fd); chassert(!error); auto config_path = std::string(tmp_file) + ".xml"; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index a6f9ce47f94..04c218cb096 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -42,6 +42,10 @@ FilterStep::FilterStep( , remove_filter_column(remove_filter_column_) { actions_dag.removeAliasesForFilter(filter_column_name); + /// Removing aliases may result in unneeded ALIAS node in DAG. + /// This should not be an issue by itself, + /// but it might trigger an issue with duplicated names in Block after plan optimizations. + actions_dag.removeUnusedActions(false, false); } void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index c0fb383d435..39ea6f03922 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -101,7 +101,8 @@ static String formattedAST(const ASTPtr & ast) return {}; WriteBufferFromOwnString buf; - IAST::FormatSettings ast_format_settings(buf, /*one_line*/ true, /*hilite*/ false, /*always_quote_identifiers*/ true); + IAST::FormatSettings ast_format_settings( + /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); ast->format(ast_format_settings); return buf.str(); } diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index c2da5753a27..78e60866afe 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -77,6 +77,14 @@ ISource::Status RemoteSource::prepare() return Status::Finished; } +#if defined(OS_LINUX) + if (async_query_sending && !was_query_sent && fd < 0) + { + startup_event_fd.write(); + return Status::Async; + } +#endif + if (is_async_state) return Status::Async; @@ -96,6 +104,15 @@ ISource::Status RemoteSource::prepare() return status; } +int RemoteSource::schedule() +{ +#if defined(OS_LINUX) + return (fd < 0 ? startup_event_fd.fd : fd); +#else + return fd; +#endif +} + void RemoteSource::work() { /// Connection drain is a heavy operation that may take a long time. diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h index c9d1a647779..59ed59b911f 100644 --- a/src/Processors/Sources/RemoteSource.h +++ b/src/Processors/Sources/RemoteSource.h @@ -5,6 +5,9 @@ #include #include + +#include + namespace DB { @@ -31,7 +34,7 @@ public: /// Stop reading from stream if output port is finished. void onUpdatePorts() override; - int schedule() override { return fd; } + int schedule() override; void onAsyncJobReady() override; @@ -42,7 +45,7 @@ protected: void onCancel() noexcept override; private: - bool was_query_sent = false; + std::atomic_bool was_query_sent = false; bool need_drain = false; bool executor_finished = false; bool add_aggregation_info = false; @@ -56,7 +59,8 @@ private: int fd = -1; size_t rows = 0; bool manually_add_rows_before_limit_counter = false; - bool preprocessed_packet = false; + std::atomic_bool preprocessed_packet = false; + EventFD startup_event_fd; }; /// Totals source from RemoteQueryExecutor. diff --git a/src/Processors/Transforms/MaterializingTransform.cpp b/src/Processors/Transforms/MaterializingTransform.cpp index 9ae80e21a68..771718e5ced 100644 --- a/src/Processors/Transforms/MaterializingTransform.cpp +++ b/src/Processors/Transforms/MaterializingTransform.cpp @@ -5,8 +5,11 @@ namespace DB { -MaterializingTransform::MaterializingTransform(const Block & header) - : ISimpleTransform(header, materializeBlock(header), false) {} +MaterializingTransform::MaterializingTransform(const Block & header, bool remove_sparse_) + : ISimpleTransform(header, materializeBlock(header), false) + , remove_sparse(remove_sparse_) +{ +} void MaterializingTransform::transform(Chunk & chunk) { @@ -14,7 +17,11 @@ void MaterializingTransform::transform(Chunk & chunk) auto columns = chunk.detachColumns(); for (auto & col : columns) - col = recursiveRemoveSparse(col->convertToFullColumnIfConst()); + { + col = col->convertToFullColumnIfConst(); + if (remove_sparse) + col = recursiveRemoveSparse(col); + } chunk.setColumns(std::move(columns), num_rows); } diff --git a/src/Processors/Transforms/MaterializingTransform.h b/src/Processors/Transforms/MaterializingTransform.h index 5ecd8522426..d384083a50d 100644 --- a/src/Processors/Transforms/MaterializingTransform.h +++ b/src/Processors/Transforms/MaterializingTransform.h @@ -8,12 +8,13 @@ namespace DB class MaterializingTransform : public ISimpleTransform { public: - explicit MaterializingTransform(const Block & header); + explicit MaterializingTransform(const Block & header, bool remove_sparse_ = true); String getName() const override { return "MaterializingTransform"; } protected: void transform(Chunk & chunk) override; + bool remove_sparse; }; } diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index 4bb3b88886e..1f6474da7d0 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -66,8 +66,7 @@ InputFormatPtr getInputFormatFromASTInsertQuery( : std::make_unique(); /// Create a source from input buffer using format from query - auto source - = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef()[Setting::max_insert_block_size]); + auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef()[Setting::max_insert_block_size]); source->addBuffer(std::move(input_buffer)); return source; } diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 9e205d4df20..7aeb7ec1bab 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -25,7 +25,6 @@ #include #include - namespace ProfileEvents { extern const Event SuspendSendingQueryToShard; diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index 83f33607dbf..6caabead544 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -260,11 +260,6 @@ private: /// Initiator identifier for distributed task processing std::shared_ptr task_iterator; - /// This is needed only for parallel reading from replicas, because - /// we create a RemoteQueryExecutor per replica and have to store additional info - /// about the number of the current replica or the count of replicas at all. - IConnections::ReplicaInfo replica_info; - /// Streams for reading from temporary tables and following sending of data /// to remote servers for GLOBAL-subqueries std::vector external_tables_data; diff --git a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp index 5a78baae53c..baef4c253c6 100644 --- a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp +++ b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp @@ -141,12 +141,12 @@ RemoteQueryExecutorReadContext::~RemoteQueryExecutorReadContext() /// connection_fd is closed by Poco::Net::Socket or Epoll if (pipe_fd[0] != -1) { - int err = close(pipe_fd[0]); + [[maybe_unused]] int err = close(pipe_fd[0]); chassert(!err || errno == EINTR); } if (pipe_fd[1] != -1) { - int err = close(pipe_fd[1]); + [[maybe_unused]] int err = close(pipe_fd[1]); chassert(!err || errno == EINTR); } } diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index b61df45133a..fe746e34805 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -1,4 +1,5 @@ #include +#include "Common/ZooKeeper/ZooKeeperConstants.h" #if USE_NURAFT @@ -93,7 +94,7 @@ struct SocketInterruptablePollWrapper socket_event.data.fd = sockfd; if (epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &socket_event) < 0) { - int err = ::close(epollfd); + [[maybe_unused]] int err = ::close(epollfd); chassert(!err || errno == EINTR); throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot insert socket into epoll queue"); @@ -102,7 +103,7 @@ struct SocketInterruptablePollWrapper pipe_event.data.fd = pipe.fds_rw[0]; if (epoll_ctl(epollfd, EPOLL_CTL_ADD, pipe.fds_rw[0], &pipe_event) < 0) { - int err = ::close(epollfd); + [[maybe_unused]] int err = ::close(epollfd); chassert(!err || errno == EINTR); throw ErrnoException(ErrorCodes::SYSTEM_ERROR, "Cannot insert socket into epoll queue"); @@ -211,7 +212,7 @@ struct SocketInterruptablePollWrapper #if defined(POCO_HAVE_FD_EPOLL) ~SocketInterruptablePollWrapper() { - int err = ::close(epollfd); + [[maybe_unused]] int err = ::close(epollfd); chassert(!err || errno == EINTR); } #endif @@ -252,7 +253,9 @@ void KeeperTCPHandler::sendHandshake(bool has_leader, bool & use_compression) Coordination::write(Coordination::SERVER_HANDSHAKE_LENGTH, *out); if (has_leader) { - if (use_compression) + if (use_xid_64) + Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64, *out); + else if (use_compression) Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION, *out); else Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION, *out); @@ -290,10 +293,23 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length, bool Coordination::read(protocol_version, *in); - if (protocol_version != Coordination::ZOOKEEPER_PROTOCOL_VERSION && protocol_version != Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION) + if (protocol_version != Coordination::ZOOKEEPER_PROTOCOL_VERSION + && protocol_version < Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION + && protocol_version > Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64) + { throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected protocol version: {}", toString(protocol_version)); + } - use_compression = (protocol_version == Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION); + if (protocol_version == Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_COMPRESSION) + { + use_compression = true; + } + else if (protocol_version >= Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_XID_64) + { + close_xid = Coordination::CLOSE_XID_64; + use_xid_64 = true; + Coordination::read(use_compression, *in); + } Coordination::read(last_zxid_seen, *in); Coordination::read(timeout_ms, *in); @@ -485,7 +501,7 @@ void KeeperTCPHandler::runImpl() updateStats(response, request_with_response.request); packageSent(); - response->write(getWriteBuffer()); + response->write(getWriteBuffer(), use_xid_64); flushWriteBuffer(); log_long_operation("Sending response"); if (response->error == Coordination::Error::ZSESSIONEXPIRED) @@ -582,8 +598,15 @@ std::pair KeeperTCPHandler::receiveReque auto & read_buffer = getReadBuffer(); int32_t length; Coordination::read(length, read_buffer); - int32_t xid; - Coordination::read(xid, read_buffer); + int64_t xid; + if (use_xid_64) + Coordination::read(xid, read_buffer); + else + { + int32_t read_xid; + Coordination::read(read_xid, read_buffer); + xid = read_xid; + } Coordination::OpNum opnum; Coordination::read(opnum, read_buffer); diff --git a/src/Server/KeeperTCPHandler.h b/src/Server/KeeperTCPHandler.h index 7c2b8acf624..5fe284ecfdd 100644 --- a/src/Server/KeeperTCPHandler.h +++ b/src/Server/KeeperTCPHandler.h @@ -84,6 +84,7 @@ private: ThreadSafeResponseQueuePtr responses; Coordination::XID close_xid = Coordination::CLOSE_XID; + bool use_xid_64 = false; /// Streams for reading/writing from/to client connection socket. std::optional in; diff --git a/src/Storages/FileLog/DirectoryWatcherBase.cpp b/src/Storages/FileLog/DirectoryWatcherBase.cpp index f1cf0866de7..338de7a1288 100644 --- a/src/Storages/FileLog/DirectoryWatcherBase.cpp +++ b/src/Storages/FileLog/DirectoryWatcherBase.cpp @@ -134,7 +134,7 @@ void DirectoryWatcherBase::watchFunc() DirectoryWatcherBase::~DirectoryWatcherBase() { stop(); - int err = ::close(inotify_fd); + [[maybe_unused]] int err = ::close(inotify_fd); chassert(!err || errno == EINTR); } diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 9222084ca0e..9d93563d739 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -558,7 +558,9 @@ StorageFileLog::ReadMetadataResult StorageFileLog::readMetadata(const String & f filename, metadata_base_path); } - auto in = disk->readFile(full_path); + auto read_settings = getReadSettings(); + read_settings.local_fs_method = LocalFSReadMethod::pread; + auto in = disk->readFile(full_path, read_settings); FileMeta metadata; UInt64 inode, last_written_pos; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 79d87a6e3cb..f56b959435c 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -31,6 +31,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int DEADLOCK_AVOIDED; extern const int CANNOT_RESTORE_TABLE; + extern const int TABLE_IS_BEING_RESTARTED; } IStorage::IStorage(StorageID storage_id_, std::unique_ptr metadata_) @@ -66,12 +67,13 @@ RWLockImpl::LockHolder IStorage::tryLockTimed( TableLockHolder IStorage::lockForShare(const String & query_id, const std::chrono::milliseconds & acquire_timeout) { TableLockHolder result = tryLockTimed(drop_lock, RWLockImpl::Read, query_id, acquire_timeout); - - if (is_dropped || is_detached) - { - auto table_id = getStorageID(); + auto table_id = getStorageID(); + if (!table_id.hasUUID() && (is_dropped || is_detached)) throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped or detached", table_id.database_name, table_id.table_name); - } + + if (is_being_restarted) + throw Exception( + ErrorCodes::TABLE_IS_BEING_RESTARTED, "Table {}.{} is being restarted", table_id.database_name, table_id.table_name); return result; } @@ -79,12 +81,10 @@ TableLockHolder IStorage::tryLockForShare(const String & query_id, const std::ch { TableLockHolder result = tryLockTimed(drop_lock, RWLockImpl::Read, query_id, acquire_timeout); - if (is_dropped || is_detached) - { - // Table was dropped while acquiring the lock + auto table_id = getStorageID(); + if (is_being_restarted || (!table_id.hasUUID() && (is_dropped || is_detached))) + // Table was dropped or is being restarted while acquiring the lock result = nullptr; - } - return result; } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 07058dfb5df..0dc48634282 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -269,6 +270,9 @@ public: /// because those are internally translated into 'ALTER UDPATE' mutations. virtual bool supportsDelete() const { return false; } + /// Returns true if storage can store columns in sparse serialization. + virtual bool supportsSparseSerialization() const { return false; } + /// Return true if the trivial count query could be optimized without reading the data at all /// in totalRows() or totalRowsByPartitionPredicate() methods or with optimized reading in read() method. /// 'storage_snapshot' may be nullptr. @@ -277,6 +281,9 @@ public: return false; } + /// Returns hints for serialization of columns accorsing to statistics accumulated by storage. + virtual SerializationInfoByName getSerializationHints() const { return {}; } + private: StorageID storage_id; diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 20833780486..797e3a32084 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -302,6 +302,7 @@ DataPartStorageOnDiskBase::getReplicatedFilesDescription(const NameSet & file_na ReplicatedFilesDescription description; auto relative_path = fs::path(root_path) / part_dir; auto disk = volume->getDisk(); + auto read_settings = getReadSettings(); auto actual_file_names = getActualFileNamesOnDisk(file_names); for (const auto & name : actual_file_names) @@ -312,9 +313,9 @@ DataPartStorageOnDiskBase::getReplicatedFilesDescription(const NameSet & file_na auto & file_desc = description.files[name]; file_desc.file_size = file_size; - file_desc.input_buffer_getter = [disk, path, file_size] + file_desc.input_buffer_getter = [disk, path, file_size, read_settings] { - return disk->readFile(path, ReadSettings{}.adjustBufferSize(file_size), file_size, file_size); + return disk->readFile(path, read_settings.adjustBufferSize(file_size), file_size, file_size); }; } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp index bc2bb020836..b3dc9ad16d4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -203,7 +204,8 @@ void DataPartStorageOnDiskFull::copyFileFrom(const IDataPartStorage & source, co source_on_disk->getDisk()->copyFile( fs::path(source_on_disk->getRelativePath()) / from, *volume->getDisk(), - fs::path(root_path) / part_dir / to); + fs::path(root_path) / part_dir / to, + getReadSettings()); } void DataPartStorageOnDiskFull::createProjection(const std::string & name) diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 6d532037806..817b63fa4dc 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace zkutil diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a1bc1342688..8c60988e7e8 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1747,7 +1747,7 @@ void IMergeTreeDataPart::appendRemovalTIDToVersionMetadata(bool clear) const static std::unique_ptr openForReading(const IDataPartStorage & part_storage, const String & filename) { size_t file_size = part_storage.getFileSize(filename); - return part_storage.readFile(filename, ReadSettings().adjustBufferSize(file_size), file_size, file_size); + return part_storage.readFile(filename, getReadSettings().adjustBufferSize(file_size), file_size, file_size); } void IMergeTreeDataPart::loadVersionMetadata() const @@ -1847,7 +1847,10 @@ bool IMergeTreeDataPart::assertHasValidVersionMetadata() const try { size_t file_size = getDataPartStorage().getFileSize(TXN_VERSION_METADATA_FILE_NAME); - auto buf = getDataPartStorage().readFile(TXN_VERSION_METADATA_FILE_NAME, ReadSettings().adjustBufferSize(file_size), file_size, std::nullopt); + auto read_settings = getReadSettings().adjustBufferSize(file_size); + /// Avoid cannot allocated thread error. No need in threadpool read method here. + read_settings.local_fs_method = LocalFSReadMethod::pread; + auto buf = getDataPartStorage().readFile(TXN_VERSION_METADATA_FILE_NAME, read_settings, file_size, std::nullopt); readStringUntilEOF(content, *buf); ReadBufferFromString str_buf{content}; diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index c87f66b64f3..4f42a7e9122 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -11,13 +12,14 @@ namespace ErrorCodes } -Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation) +Block getIndexBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation) { Block result; for (size_t i = 0, size = names.size(); i < size; ++i) { - const auto & name = names[i]; - result.insert(i, block.getByName(name)); + auto src_column = block.getByName(names[i]); + src_column.column = recursiveRemoveSparse(src_column.column); + result.insert(i, src_column); /// Reorder primary key columns in advance and add them to `primary_key_columns`. if (permutation) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 2fdb0794789..eb51a1b2922 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -16,7 +16,7 @@ namespace DB struct MergeTreeSettings; using MergeTreeSettingsPtr = std::shared_ptr; -Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); +Block getIndexBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); Block permuteBlockIfNeeded(const Block & block, const IColumn::Permutation * permutation); diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 1f46d6b8e1b..b2f18f08f41 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -172,7 +172,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns if (dag) { - dag->addMaterializingOutputActions(); + dag->addMaterializingOutputActions(/*materialize_sparse=*/ false); auto actions = std::make_shared( std::move(*dag), ExpressionActionsSettings::fromSettings(data_part_info_for_read->getContext()->getSettingsRef())); diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 307dd81ecd4..da05cdb2206 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1,10 +1,10 @@ #include #include +#include #include #include #include #include -#include #include #include #include @@ -17,8 +17,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -28,18 +30,20 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include #include +#include +#include +#include + namespace DB { @@ -459,12 +463,20 @@ const KeyCondition::AtomMap KeyCondition::atom_map out.range = Range::createWholeUniverseWithoutNull(); return true; } + }, + { + "pointInPolygon", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_POINT_IN_POLYGON; + return true; + } } }; static const std::set always_relaxed_atom_functions = {"match"}; static const std::set always_relaxed_atom_elements - = {KeyCondition::RPNElement::FUNCTION_UNKNOWN, KeyCondition::RPNElement::FUNCTION_ARGS_IN_HYPERRECTANGLE}; + = {KeyCondition::RPNElement::FUNCTION_UNKNOWN, KeyCondition::RPNElement::FUNCTION_ARGS_IN_HYPERRECTANGLE, KeyCondition::RPNElement::FUNCTION_POINT_IN_POLYGON}; /// Functions with range inversion cannot be relaxed. It will become stricter instead. /// For example: @@ -1850,6 +1862,54 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, RPNEleme if (atom_map.find(func_name) == std::end(atom_map)) return false; + auto analyze_point_in_polygon = [&, this]() -> bool + { + /// pointInPolygon((x, y), [(0, 0), (8, 4), (5, 8), (0, 2)]) + if (func.getArgumentAt(0).tryGetConstant(const_value, const_type)) + return false; + if (!func.getArgumentAt(1).tryGetConstant(const_value, const_type)) + return false; + + const auto atom_it = atom_map.find(func_name); + + /// Analyze (x, y) + RPNElement::MultiColumnsFunctionDescription column_desc; + column_desc.function_name = func_name; + auto first_argument = func.getArgumentAt(0).toFunctionNode(); + + if (first_argument.getArgumentsSize() != 2 || first_argument.getFunctionName() != "tuple") + return false; + + for (size_t i = 0; i < 2; ++i) + { + auto name = first_argument.getArgumentAt(i).getColumnName(); + auto it = key_columns.find(name); + if (it == key_columns.end()) + return false; + column_desc.key_columns.push_back(name); + column_desc.key_column_positions.push_back(key_columns[name]); + } + out.point_in_polygon_column_description = column_desc; + + /// Analyze [(0, 0), (8, 4), (5, 8), (0, 2)] + chassert(WhichDataType(const_type).isArray()); + for (const auto & elem : const_value.safeGet()) + { + if (elem.getType() != Field::Types::Tuple) + return false; + + const auto & elem_tuple = elem.safeGet(); + if (elem_tuple.size() != 2) + return false; + + auto x = applyVisitor(FieldVisitorConvertToNumber(), elem_tuple[0]); + auto y = applyVisitor(FieldVisitorConvertToNumber(), elem_tuple[1]); + out.polygon.outer().push_back({x, y}); + } + boost::geometry::correct(out.polygon); + return atom_it->second(out, const_value); + }; + if (always_relaxed_atom_functions.contains(func_name)) relaxed = true; @@ -1879,6 +1939,11 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, RPNEleme else return false; } + else if (func_name == "pointInPolygon") + { + /// Case1 no holes in polygon + return analyze_point_in_polygon(); + } else if (func.getArgumentAt(1).tryGetConstant(const_value, const_type)) { /// If the const operand is null, the atom will be always false @@ -2047,7 +2112,15 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, RPNEleme } else - return false; + { + if (func_name == "pointInPolygon") + { + /// Case2 has holes in polygon, when checking skip index, the hole will be ignored. + return analyze_point_in_polygon(); + } + else + return false; + } const auto atom_it = atom_map.find(func_name); @@ -3059,6 +3132,46 @@ BoolMask KeyCondition::checkInHyperrectangle( * represented by a set of hyperrectangles. */ } + else if (element.function == RPNElement::FUNCTION_POINT_IN_POLYGON) + { + /** There are 2 kinds of polygons: + * 1. Polygon by minmax index + * 2. Polygons which is provided by user + * + * Polygon by minmax index: + * For hyperactangle [1, 2] × [3, 4] we can create a polygon with 4 points: (1, 3), (1, 4), (2, 4), (2, 3) + * + * Algorithm: + * Check whether there is any intersection of the 2 polygons. If true return {true, true}, else return {false, true}. + */ + const auto & key_column_positions = element.point_in_polygon_column_description->key_column_positions; + + Float64 x_min = applyVisitor(FieldVisitorConvertToNumber(), hyperrectangle[key_column_positions[0]].left); + Float64 x_max = applyVisitor(FieldVisitorConvertToNumber(), hyperrectangle[key_column_positions[0]].right); + Float64 y_min = applyVisitor(FieldVisitorConvertToNumber(), hyperrectangle[key_column_positions[1]].left); + Float64 y_max = applyVisitor(FieldVisitorConvertToNumber(), hyperrectangle[key_column_positions[1]].right); + + if (unlikely(isNaN(x_min) || isNaN(x_max) || isNaN(y_min) || isNaN(y_max))) + { + rpn_stack.emplace_back(true, true); + continue; + } + + using Point = boost::geometry::model::d2::point_xy; + using Polygon = boost::geometry::model::polygon; + Polygon polygon_by_minmax_index; + polygon_by_minmax_index.outer().emplace_back(x_min, y_min); + polygon_by_minmax_index.outer().emplace_back(x_min, y_max); + polygon_by_minmax_index.outer().emplace_back(x_max, y_max); + polygon_by_minmax_index.outer().emplace_back(x_max, y_min); + + /// Close ring + boost::geometry::correct(polygon_by_minmax_index); + + /// Because the polygon may have a hole so the "can_be_false" should always be true. + rpn_stack.emplace_back( + boost::geometry::intersects(polygon_by_minmax_index, element.polygon), true); + } else if ( element.function == RPNElement::FUNCTION_IS_NULL || element.function == RPNElement::FUNCTION_IS_NOT_NULL) @@ -3138,7 +3251,16 @@ bool KeyCondition::mayBeTrueInRange( String KeyCondition::RPNElement::toString() const { if (argument_num_of_space_filling_curve) - return toString(fmt::format("argument {} of column {}", *argument_num_of_space_filling_curve, key_column), true); + return toString(fmt::format("argument {} of column {}", *argument_num_of_space_filling_curve, key_column), false); + else if (point_in_polygon_column_description) + { + return toString( + fmt::format( + "column ({}, {})", + point_in_polygon_column_description->key_columns[0], + point_in_polygon_column_description->key_columns[1]), + false); + } else return toString(fmt::format("column {}", key_column), true); } @@ -3218,6 +3340,23 @@ String KeyCondition::RPNElement::toString(std::string_view column_name, bool pri buf << ")"; return buf.str(); } + case FUNCTION_POINT_IN_POLYGON: + { + auto points_in_polygon = polygon.outer(); + buf << "("; + print_wrapped_column(buf); + buf << " in "; + buf << "["; + for (size_t i = 0; i < points_in_polygon.size(); ++i) + { + if (i != 0) + buf << ", "; + buf << "(" << points_in_polygon[i].x() << ", " << points_in_polygon[i].y() << ")"; + } + buf << "]"; + buf << ")"; + return buf.str(); + } case FUNCTION_IS_NULL: case FUNCTION_IS_NOT_NULL: { @@ -3269,6 +3408,7 @@ bool KeyCondition::unknownOrAlwaysTrue(bool unknown_any) const || element.function == RPNElement::FUNCTION_IN_SET || element.function == RPNElement::FUNCTION_NOT_IN_SET || element.function == RPNElement::FUNCTION_ARGS_IN_HYPERRECTANGLE + || element.function == RPNElement::FUNCTION_POINT_IN_POLYGON || element.function == RPNElement::FUNCTION_IS_NULL || element.function == RPNElement::FUNCTION_IS_NOT_NULL || element.function == RPNElement::ALWAYS_FALSE) diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 8bbb86aba43..8c946bd3bbd 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -2,14 +2,13 @@ #include +#include + #include #include -#include #include -#include - #include #include #include @@ -168,6 +167,9 @@ public: /// this expression will be analyzed and then represented by following: /// args in hyperrectangle [10, 20] × [20, 30]. FUNCTION_ARGS_IN_HYPERRECTANGLE, + /// Special for pointInPolygon to utilize minmax indices. + /// For example: pointInPolygon((x, y), [(0, 0), (0, 2), (2, 2), (2, 0)]) + FUNCTION_POINT_IN_POLYGON, /// Can take any value. FUNCTION_UNKNOWN, /// Operators of the logical expression. @@ -206,6 +208,21 @@ public: /// For FUNCTION_ARGS_IN_HYPERRECTANGLE Hyperrectangle space_filling_curve_args_hyperrectangle; + /// For FUNCTION_POINT_IN_POLYGON. + /// Function like 'pointInPolygon' has multiple columns. + /// This struct description column part of the function, such as (x, y) in 'pointInPolygon'. + struct MultiColumnsFunctionDescription + { + String function_name; + std::vector key_column_positions; + std::vector key_columns; + }; + std::optional point_in_polygon_column_description; + + using Point = boost::geometry::model::d2::point_xy; + using Polygon = boost::geometry::model::polygon; + Polygon polygon; + MonotonicFunctionsChain monotonic_functions_chain; }; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index d4c91467cd4..f0447e71539 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -187,6 +187,20 @@ static void addMissedColumnsToSerializationInfos( } } +bool MergeTask::GlobalRuntimeContext::isCancelled() const +{ + return (future_part ? merges_blocker->isCancelledForPartition(future_part->part_info.partition_id) : merges_blocker->isCancelled()) + || merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed); +} + +void MergeTask::GlobalRuntimeContext::checkOperationIsNotCanceled() const +{ + if (isCancelled()) + { + throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); + } +} + /// PK columns are sorted and merged, ordinary columns are gathered using info from merge step void MergeTask::ExecuteAndFinalizeHorizontalPart::extractMergingAndGatheringColumns() const { @@ -280,8 +294,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const const String local_tmp_suffix = global_ctx->parent_part ? global_ctx->suffix : ""; - if (global_ctx->merges_blocker->isCancelled() || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed)) - throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); + global_ctx->checkOperationIsNotCanceled(); /// We don't want to perform merge assigned with TTL as normal merge, so /// throw exception @@ -526,9 +539,10 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const ctx->is_cancelled = [merges_blocker = global_ctx->merges_blocker, ttl_merges_blocker = global_ctx->ttl_merges_blocker, need_remove = ctx->need_remove_expired_values, - merge_list_element = global_ctx->merge_list_element_ptr]() -> bool + merge_list_element = global_ctx->merge_list_element_ptr, + partition_id = global_ctx->future_part->part_info.partition_id]() -> bool { - return merges_blocker->isCancelled() + return merges_blocker->isCancelledForPartition(partition_id) || (need_remove && ttl_merges_blocker->isCancelled()) || merge_list_element->is_cancelled.load(std::memory_order_relaxed); }; @@ -805,8 +819,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::finalize() const global_ctx->merging_executor.reset(); global_ctx->merged_pipeline.reset(); - if (global_ctx->merges_blocker->isCancelled() || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed)) - throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); + global_ctx->checkOperationIsNotCanceled(); if (ctx->need_remove_expired_values && global_ctx->ttl_merges_blocker->isCancelled()) throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts with expired TTL"); @@ -995,7 +1008,7 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic indexes_to_recalc = MergeTreeIndexFactory::instance().getMany(indexes_it->second); auto indices_expression_dag = indexes_it->second.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); - indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. + indices_expression_dag.addMaterializingOutputActions(/*materialize_sparse=*/ true); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_column_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); @@ -1069,8 +1082,7 @@ bool MergeTask::VerticalMergeStage::executeVerticalMergeForOneColumn() const { Block block; - if (global_ctx->merges_blocker->isCancelled() - || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed) + if (global_ctx->isCancelled() || !ctx->executor->pull(block)) return false; @@ -1086,8 +1098,7 @@ bool MergeTask::VerticalMergeStage::executeVerticalMergeForOneColumn() const void MergeTask::VerticalMergeStage::finalizeVerticalMergeForOneColumn() const { const String & column_name = ctx->it_name_and_type->name; - if (global_ctx->merges_blocker->isCancelled() || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed)) - throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); + global_ctx->checkOperationIsNotCanceled(); ctx->executor.reset(); auto changed_checksums = ctx->column_to->fillChecksums(global_ctx->new_data_part, global_ctx->checksums_gathered_columns); @@ -1719,7 +1730,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (!global_ctx->merging_skip_indexes.empty()) { auto indices_expression_dag = global_ctx->merging_skip_indexes.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); - indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. + indices_expression_dag.addMaterializingOutputActions(/*materialize_sparse=*/ true); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_parts_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 29b5c4452e7..5a4fb1ec0b8 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -27,6 +27,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -86,7 +87,7 @@ public: MergeTreeTransactionPtr txn, MergeTreeData * data_, MergeTreeDataMergerMutator * mutator_, - ActionBlocker * merges_blocker_, + PartitionActionBlocker * merges_blocker_, ActionBlocker * ttl_merges_blocker_) { global_ctx = std::make_shared(); @@ -161,7 +162,7 @@ private: MergeListElement * merge_list_element_ptr{nullptr}; MergeTreeData * data{nullptr}; MergeTreeDataMergerMutator * mutator{nullptr}; - ActionBlocker * merges_blocker{nullptr}; + PartitionActionBlocker * merges_blocker{nullptr}; ActionBlocker * ttl_merges_blocker{nullptr}; StorageSnapshotPtr storage_snapshot{nullptr}; StorageMetadataPtr metadata_snapshot{nullptr}; @@ -215,7 +216,12 @@ private: MergeTreeData::MergingParams merging_params{}; scope_guard temporary_directory_lock; + UInt64 prev_elapsed_ms{0}; + + // will throw an exception if merge was cancelled in any way. + void checkOperationIsNotCanceled() const; + bool isCancelled() const; }; using GlobalRuntimeContextPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c728ffd63d4..a0f3862c287 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -295,7 +295,7 @@ void MergeTreeData::initializeDirectoriesAndFormatVersion(const std::string & re if (disk->exists(format_version_path)) { - auto buf = disk->readFile(format_version_path); + auto buf = disk->readFile(format_version_path, getReadSettings()); UInt32 current_format_version{0}; readIntText(current_format_version, *buf); if (!buf->eof()) @@ -1904,6 +1904,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optionalrenameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes resetObjectColumnsFromActiveParts(part_lock); + resetSerializationHints(part_lock); calculateColumnAndSecondaryIndexSizesImpl(); PartLoadingTreeNodes unloaded_parts; @@ -6908,6 +6910,8 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(DataPartsLock } } + data.updateSerializationHints(precommitted_parts, total_covered_parts, parts_lock); + if (reduce_parts == 0) { for (const auto & part : precommitted_parts) @@ -8571,6 +8575,66 @@ void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPart DB::updateObjectColumns(object_columns, columns, part->getColumns()); } +template +static void updateSerializationHintsForPart(const DataPartPtr & part, const ColumnsDescription & storage_columns, SerializationInfoByName & hints, bool remove) +{ + const auto & part_columns = part->getColumnsDescription(); + for (const auto & [name, info] : part->getSerializationInfos()) + { + auto new_hint = hints.tryGet(name); + if (!new_hint) + continue; + + /// Structure may change after alter. Do not add info for such items. + /// Instead it will be updated on commit of the result part of alter. + if (part_columns.tryGetPhysical(name) != storage_columns.tryGetPhysical(name)) + continue; + + chassert(new_hint->structureEquals(*info)); + if (remove) + new_hint->remove(*info); + else + new_hint->add(*info); + } +} + +void MergeTreeData::resetSerializationHints(const DataPartsLock & /*lock*/) +{ + SerializationInfo::Settings settings = + { + .ratio_of_defaults_for_sparse = getSettings()->ratio_of_defaults_for_sparse_serialization, + .choose_kind = true, + }; + + const auto & storage_columns = getInMemoryMetadataPtr()->getColumns(); + serialization_hints = SerializationInfoByName(storage_columns.getAllPhysical(), settings); + auto range = getDataPartsStateRange(DataPartState::Active); + + for (const auto & part : range) + updateSerializationHintsForPart(part, storage_columns, serialization_hints, false); +} + +template +void MergeTreeData::updateSerializationHints(const AddedParts & added_parts, const RemovedParts & removed_parts, const DataPartsLock & /*lock*/) +{ + const auto & storage_columns = getInMemoryMetadataPtr()->getColumns(); + + for (const auto & part : added_parts) + updateSerializationHintsForPart(part, storage_columns, serialization_hints, false); + + for (const auto & part : removed_parts) + updateSerializationHintsForPart(part, storage_columns, serialization_hints, true); +} + +SerializationInfoByName MergeTreeData::getSerializationHints() const +{ + auto lock = lockParts(); + SerializationInfoByName res; + for (const auto & [name, info] : serialization_hints) + res.emplace(name, info->clone()); + return res; +} + bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const { if (hasLightweightDeletedMask()) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 5edd24db40d..7a9730e8627 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -441,6 +441,7 @@ public: bool supportsDynamicSubcolumnsDeprecated() const override { return true; } bool supportsDynamicSubcolumns() const override { return true; } + bool supportsSparseSerialization() const override { return true; } bool supportsLightweightDelete() const override; @@ -1242,6 +1243,11 @@ protected: /// protected by @data_parts_mutex. ColumnsDescription object_columns; + /// Serialization info accumulated among all active parts. + /// It changes only when set of parts is changed and is + /// protected by @data_parts_mutex. + SerializationInfoByName serialization_hints; + MergeTreePartsMover parts_mover; /// Executors are common for both ReplicatedMergeTree and plain MergeTree @@ -1530,6 +1536,13 @@ protected: void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); + void resetSerializationHints(const DataPartsLock & lock); + + template + void updateSerializationHints(const AddedParts & added_parts, const RemovedParts & removed_parts, const DataPartsLock & lock); + + SerializationInfoByName getSerializationHints() const override; + /** A structure that explicitly represents a "merge tree" of parts * which is implicitly presented by min-max block numbers and levels of parts. * The children of node are parts which are covered by parent part. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index c14962d903b..ca3fae7ec90 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -207,7 +207,7 @@ public : /** Is used to cancel all merges and mutations. On cancel() call all currently running actions will throw exception soon. * All new attempts to start a merge or mutation will throw an exception until all 'LockHolder' objects will be destroyed. */ - ActionBlocker merges_blocker; + PartitionActionBlocker merges_blocker; ActionBlocker ttl_merges_blocker; private: diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 5bab523a9f1..d9d4b6e4b8f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -136,7 +136,7 @@ void MergeTreeDataPartWide::loadIndexGranularityImpl( } else { - auto marks_file = data_part_storage_.readFile(marks_file_path, ReadSettings().adjustBufferSize(marks_file_size), marks_file_size, std::nullopt); + auto marks_file = data_part_storage_.readFile(marks_file_path, getReadSettings().adjustBufferSize(marks_file_size), marks_file_size, std::nullopt); std::unique_ptr marks_reader; if (!index_granularity_info_.mark_type.compressed) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index f4be7619fc8..a859172023f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -213,11 +213,11 @@ void MergeTreeDataPartWriterCompact::writeDataBlockPrimaryIndexAndSkipIndices(co if (settings.rewrite_primary_key) { - Block primary_key_block = getBlockAndPermute(block, metadata_snapshot->getPrimaryKeyColumns(), nullptr); + Block primary_key_block = getIndexBlockAndPermute(block, metadata_snapshot->getPrimaryKeyColumns(), nullptr); calculateAndSerializePrimaryIndex(primary_key_block, granules_to_write); } - Block skip_indices_block = getBlockAndPermute(block, getSkipIndicesColumns(), nullptr); + Block skip_indices_block = getIndexBlockAndPermute(block, getSkipIndicesColumns(), nullptr); calculateAndSerializeSkipIndices(skip_indices_block, granules_to_write); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index f050accd7a1..04e07a0588a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -296,9 +296,9 @@ void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Perm auto offset_columns = written_offset_columns ? *written_offset_columns : WrittenOffsetColumns{}; Block primary_key_block; if (settings.rewrite_primary_key) - primary_key_block = getBlockAndPermute(block, metadata_snapshot->getPrimaryKeyColumns(), permutation); + primary_key_block = getIndexBlockAndPermute(block, metadata_snapshot->getPrimaryKeyColumns(), permutation); - Block skip_indexes_block = getBlockAndPermute(block, getSkipIndicesColumns(), permutation); + Block skip_indexes_block = getIndexBlockAndPermute(block, getSkipIndicesColumns(), permutation); auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index ed68200041b..130d9ca8f6a 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -577,6 +577,13 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( SerializationInfoByName infos(columns, settings); infos.add(block); + for (const auto & [column_name, _] : columns) + { + auto & column = block.getByName(column_name); + if (column.column->isSparse() && infos.getKind(column_name) != ISerialization::Kind::SPARSE) + column.column = recursiveRemoveSparse(column.column); + } + new_data_part->setColumns(columns, infos, metadata_snapshot->getMetadataVersion()); new_data_part->rows_count = block.rows(); new_data_part->existing_rows_count = block.rows(); diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp index a8110500f13..5d1bf0294bb 100644 --- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp +++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp @@ -140,7 +140,7 @@ void MergeTreeDeduplicationLog::load() size_t MergeTreeDeduplicationLog::loadSingleLog(const std::string & path) { - auto read_buf = disk->readFile(path); + auto read_buf = disk->readFile(path, getReadSettings()); size_t total_entries = 0; while (!read_buf->eof()) diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 5970aed497e..9f79b282a1c 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -123,7 +123,7 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & pat , is_temp(false) { block_number = parseFileName(file_name); - auto buf = disk->readFile(path_prefix + file_name); + auto buf = disk->readFile(path_prefix + file_name, getReadSettings()); *buf >> "format version: 1\n"; diff --git a/src/Storages/MergeTree/MergeTreeReadTask.h b/src/Storages/MergeTree/MergeTreeReadTask.h index e90a07e0b55..748babb5b4c 100644 --- a/src/Storages/MergeTree/MergeTreeReadTask.h +++ b/src/Storages/MergeTree/MergeTreeReadTask.h @@ -66,7 +66,7 @@ struct MergeTreeReadTaskInfo MergeTreeReadTaskColumns task_columns; /// Shared initialized size predictor. It is copied for each new task. MergeTreeBlockSizePredictorPtr shared_size_predictor; - /// TODO: comment + /// Shared constant fields for virtual columns. VirtualFields const_virtual_fields; /// The amount of data to read per task based on size of the queried columns. size_t min_marks_per_task = 0; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index c526cebd22d..9b5d3176323 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -70,14 +71,6 @@ namespace ErrorCodes namespace MutationHelpers { -static bool checkOperationIsNotCanceled(ActionBlocker & merges_blocker, MergeListEntry * mutate_entry) -{ - if (merges_blocker.isCancelled() || (*mutate_entry)->is_cancelled) - throw Exception(ErrorCodes::ABORTED, "Cancelled mutating parts"); - - return true; -} - static bool haveMutationsOfDynamicColumns(const MergeTreeData::DataPartPtr & data_part, const MutationCommands & commands) { for (const auto & command : commands) @@ -1001,7 +994,7 @@ struct MutationContext { MergeTreeData * data; MergeTreeDataMergerMutator * mutator; - ActionBlocker * merges_blocker; + PartitionActionBlocker * merges_blocker; TableLockHolder * holder; MergeListEntry * mutate_entry; @@ -1065,6 +1058,17 @@ struct MutationContext scope_guard temporary_directory_lock; + bool checkOperationIsNotCanceled() const + { + if (new_data_part ? merges_blocker->isCancelledForPartition(new_data_part->info.partition_id) : merges_blocker->isCancelled() + || (*mutate_entry)->is_cancelled) + { + throw Exception(ErrorCodes::ABORTED, "Cancelled mutating parts"); + } + + return true; + } + /// Whether we need to count lightweight delete rows in this mutation bool count_lightweight_deleted_rows; UInt64 execute_elapsed_ns = 0; @@ -1072,7 +1076,6 @@ struct MutationContext using MutationContextPtr = std::shared_ptr; - // This class is responsible for: // 1. get projection pipeline and a sink to write parts // 2. build an executor that can write block to the input stream (actually we can write through it to generate as many parts as possible) @@ -1202,9 +1205,7 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() Block cur_block; Block projection_header; - MutationHelpers::checkOperationIsNotCanceled(*ctx->merges_blocker, ctx->mutate_entry); - - if (!ctx->mutating_executor->pull(cur_block)) + if (!ctx->checkOperationIsNotCanceled() || !ctx->mutating_executor->pull(cur_block)) { finalizeTempProjections(); return false; @@ -1989,7 +1990,7 @@ MutateTask::MutateTask( const MergeTreeTransactionPtr & txn, MergeTreeData & data_, MergeTreeDataMergerMutator & mutator_, - ActionBlocker & merges_blocker_, + PartitionActionBlocker & merges_blocker_, bool need_prefix_) : ctx(std::make_shared()) { @@ -2031,7 +2032,7 @@ bool MutateTask::execute() } case State::NEED_EXECUTE: { - MutationHelpers::checkOperationIsNotCanceled(*ctx->merges_blocker, ctx->mutate_entry); + ctx->checkOperationIsNotCanceled(); if (task->executeStep()) return true; @@ -2124,7 +2125,7 @@ static bool canSkipMutationCommandForPart(const MergeTreeDataPartPtr & part, con bool MutateTask::prepare() { ProfileEvents::increment(ProfileEvents::MutationTotalParts); - MutationHelpers::checkOperationIsNotCanceled(*ctx->merges_blocker, ctx->mutate_entry); + ctx->checkOperationIsNotCanceled(); if (ctx->future_part->parts.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to mutate {} parts, not one. " diff --git a/src/Storages/MergeTree/MutateTask.h b/src/Storages/MergeTree/MutateTask.h index 08427bff6d8..0141f53d26d 100644 --- a/src/Storages/MergeTree/MutateTask.h +++ b/src/Storages/MergeTree/MutateTask.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,7 @@ public: const MergeTreeTransactionPtr & txn, MergeTreeData & data_, MergeTreeDataMergerMutator & mutator_, - ActionBlocker & merges_blocker_, + PartitionActionBlocker & merges_blocker_, bool need_prefix_); bool execute(); diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index a5730049907..62115015817 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -754,7 +754,12 @@ size_t DefaultCoordinator::computeConsistentHash(const std::string & part_name, ParallelReadResponse DefaultCoordinator::handleRequest(ParallelReadRequest request) { - LOG_TRACE(log, "Handling request from replica {}, minimal marks size is {}", request.replica_num, request.min_number_of_marks); + LOG_TRACE( + log, + "Handling request from replica {}, minimal marks size is {}, request count {}", + request.replica_num, + request.min_number_of_marks, + stats[request.replica_num].number_of_requests); ParallelReadResponse response; diff --git a/src/Storages/MergeTree/PartMetadataManagerOrdinary.cpp b/src/Storages/MergeTree/PartMetadataManagerOrdinary.cpp index 30823d593a2..149ad6b4a10 100644 --- a/src/Storages/MergeTree/PartMetadataManagerOrdinary.cpp +++ b/src/Storages/MergeTree/PartMetadataManagerOrdinary.cpp @@ -11,7 +11,7 @@ namespace DB std::unique_ptr PartMetadataManagerOrdinary::read(const String & file_name) const { size_t file_size = part->getDataPartStorage().getFileSize(file_name); - auto res = part->getDataPartStorage().readFile(file_name, ReadSettings().adjustBufferSize(file_size), file_size, std::nullopt); + auto res = part->getDataPartStorage().readFile(file_name, getReadSettings().adjustBufferSize(file_size), file_size, std::nullopt); if (isCompressedFromFileName(file_name)) return std::make_unique(std::move(res)); diff --git a/src/Storages/MergeTree/PartitionActionBlocker.cpp b/src/Storages/MergeTree/PartitionActionBlocker.cpp new file mode 100644 index 00000000000..4a4e7892f57 --- /dev/null +++ b/src/Storages/MergeTree/PartitionActionBlocker.cpp @@ -0,0 +1,86 @@ +#include + +#include +#include +#include + +namespace DB +{ + +ActionLock PartitionActionBlocker::cancelForPartition(const std::string & partition_id) +{ + std::unique_lock lock(mutex); + const size_t prev_size = partition_blockers.size(); + + ActionLock result = partition_blockers[partition_id].cancel(); + + /// Cleanup stale `ActionBlocker` instances once in a while, to prevent unbound growth. + ++cleanup_counter; + if (prev_size != partition_blockers.size() && cleanup_counter > 32) // 32 is arbitrary. + compactPartitionBlockersLocked(); + + return result; +} + +bool PartitionActionBlocker::isCancelledForPartition(const std::string & partition_id) const +{ + if (isCancelled()) + return true; + + std::shared_lock lock(mutex); + return isCancelledForPartitionOnlyLocked(partition_id); +} + +bool PartitionActionBlocker::isCancelledForPartitionOnlyLocked(const std::string & partition_id) const +{ + auto p = partition_blockers.find(partition_id); + return p != partition_blockers.end() && p->second.isCancelled(); +} + +size_t PartitionActionBlocker::countPartitionBlockers() const +{ + std::shared_lock lock(mutex); + return partition_blockers.size(); +} + +void PartitionActionBlocker::compactPartitionBlockers() +{ + std::unique_lock lock(mutex); + + compactPartitionBlockersLocked(); +} + +void PartitionActionBlocker::compactPartitionBlockersLocked() +{ + std::erase_if(partition_blockers, [](const auto & p) + { + return !p.second.isCancelled(); + }); + cleanup_counter = 0; +} + +std::string PartitionActionBlocker::formatDebug() const +{ + std::shared_lock lock(mutex); + + WriteBufferFromOwnString out; + out << "Global lock: " << global_blocker.getCounter().load() + << "\n" + << partition_blockers.size() << " live partition locks: {"; + + size_t i = 0; + for (const auto & p : partition_blockers) + { + out << "\n\t" << DB::double_quote << p.first << ": " << p.second.getCounter().load(); + + if (++i < partition_blockers.size()) + out << ", "; + else + out << "\n"; + } + out << "}"; + + return out.str(); +} + +} diff --git a/src/Storages/MergeTree/PartitionActionBlocker.h b/src/Storages/MergeTree/PartitionActionBlocker.h new file mode 100644 index 00000000000..edabd829b64 --- /dev/null +++ b/src/Storages/MergeTree/PartitionActionBlocker.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +/// Locks actions on given MergeTree-family table. +/// Like, ActionBlocker, but has two modes: +/// - 'partition-specific' lock, @see `cancelForPartition()`, and `isCancelledForPartition()`. +/// Only actions on *specific partition* are prohibited to start +/// and must interrupt at firsts chance. +/// There could be arbitrary number of partitions locked simultaneously. +/// +/// - 'global' lock, @see `cancel()`, `isCancelled()`, and `cancelForever()` +/// Any new actions for *ALL* partitions are prohibited to start +/// and are required to interrupt at first possible moment. +/// As if all individual partitions were locked with 'partition-specific lock'. +/// +/// Any lock can be set multiple times and considered fully un-locked +/// only when it was un-locked same number of times (by destructing/resetting of `ActionLock`). +/// +/// There could be any number of locks active at given point on single ActionBlocker instance: +/// - 'global lock' locked `N` times. +/// - `M` 'partition lock's each locked different number of times. +class PartitionActionBlocker +{ +public: + PartitionActionBlocker() = default; + + bool isCancelled() const { return global_blocker.isCancelled(); } + bool isCancelledForPartition(const std::string & /*partition_id*/) const; + + /// Temporarily blocks corresponding actions (while the returned object is alive) + friend class ActionLock; + ActionLock cancel() { return ActionLock(global_blocker); } + ActionLock cancelForPartition(const std::string & partition_id); + + /// Cancel the actions forever. + void cancelForever() { global_blocker.cancelForever(); } + + const std::atomic & getCounter() const { return global_blocker.getCounter(); } + + size_t countPartitionBlockers() const; + + /// Cleanup partition blockers + void compactPartitionBlockers(); + + std::string formatDebug() const; + +private: + void compactPartitionBlockersLocked(); + bool isCancelledForPartitionOnlyLocked(const std::string & partition_id) const; + + ActionBlocker global_blocker; + + mutable std::shared_mutex mutex; + std::unordered_map partition_blockers; + size_t cleanup_counter = 0; +}; + + +} diff --git a/src/Storages/MergeTree/tests/gtest_partition_action_blocker.cpp b/src/Storages/MergeTree/tests/gtest_partition_action_blocker.cpp new file mode 100644 index 00000000000..45a326e3059 --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_partition_action_blocker.cpp @@ -0,0 +1,257 @@ +#include + +#include +#include +#include + +#include + +#include + +using namespace DB; + +TEST(PartitionActionBlocker, TestDefaultConstructor) +{ + PartitionActionBlocker blocker; + + EXPECT_FALSE(blocker.isCancelled()); + // EXPECT_EQ(0, blocker.getCounter().load()); +} + +TEST(PartitionActionBlocker, TestCancelForever) +{ + PartitionActionBlocker blocker; + + blocker.cancelForever(); + EXPECT_TRUE(blocker.isCancelled()); + // EXPECT_EQ(1, blocker.getCounter().load()); +} + +TEST(PartitionActionBlocker, TestCancel) +{ + PartitionActionBlocker blocker; + + { + auto lock = blocker.cancel(); + EXPECT_TRUE(blocker.isCancelled()); + // EXPECT_EQ(1, blocker.getCounter().load()); + } + // automatically un-cancelled on `lock` destruction + EXPECT_FALSE(blocker.isCancelled()); +} + +TEST(PartitionActionBlocker, TestCancelForPartition) +{ + const std::string partition_id = "some partition id"; + const std::string partition_id2 = "some other partition id"; + PartitionActionBlocker blocker; + + { + auto lock = blocker.cancelForPartition(partition_id); + + // blocker is not locked fully + EXPECT_FALSE(blocker.isCancelled()); + // EXPECT_EQ(0, blocker.getCounter().load()); + + // blocker reports that only partition is locked + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + + // doesn't affect other partitions + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); + } + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id)); +} + +TEST(PartitionActionBlocker, TestCancelForTwoPartitions) +{ + const std::string partition_id1 = "some partition id"; + const std::string partition_id2 = "some other partition id"; + PartitionActionBlocker blocker; + + { + auto lock1 = blocker.cancelForPartition(partition_id1); + auto lock2 = blocker.cancelForPartition(partition_id2); + + // blocker is not locked fully + EXPECT_FALSE(blocker.isCancelled()); + // EXPECT_EQ(0, blocker.getCounter().load()); + + // blocker reports that both partitions are locked + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id1)); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id2)); + } + + // blocker is not locked fully + EXPECT_FALSE(blocker.isCancelled()); + // EXPECT_EQ(0, blocker.getCounter().load()); + + // blocker reports that only partition is locked + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id1)); +} + +TEST(PartitionActionBlocker, TestCancelForSamePartitionTwice) +{ + // Partition is unlocked only when all locks are destroyed. + + const std::string partition_id = "some partition id"; + const std::string partition_id2 = "some other partition id"; + + // Lock `partition_id` twice, make sure that global lock + // and other partitions are unaffected. + // Check that `partition_id` is unlocked only after both locks are destroyed. + + PartitionActionBlocker blocker; + + { + auto lock1 = blocker.cancelForPartition(partition_id); + { + auto lock2 = blocker.cancelForPartition(partition_id); + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); + } + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); + } + // All locks lifted + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id)); +} + +TEST(PartitionActionBlocker, TestCancelAndThenCancelForPartition) +{ + // Partition is unlocked only when all locks are destroyed. + + const std::string partition_id = "some partition id"; + const std::string partition_id2 = "some other partition id"; + PartitionActionBlocker blocker; + + { + auto global_lock = blocker.cancel(); + + { + auto lock1 = blocker.cancelForPartition(partition_id); + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id2)); + } + + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id2)); + } + // All locks lifted + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id)); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); +} + + +TEST(PartitionActionBlocker, TestCancelForPartitionAndThenCancel) +{ + // Partition is unlocked only when all locks are destroyed. + + const std::string partition_id = "some partition id"; + const std::string partition_id2 = "some other partition id"; + PartitionActionBlocker blocker; + + { + auto lock1 = blocker.cancelForPartition(partition_id); + { + auto global_lock = blocker.cancel(); + + EXPECT_TRUE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id2)); + } + + // global_locked is 'no more', so only 1 partition is locked now. + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_TRUE(blocker.isCancelledForPartition(partition_id)); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); + } + // All locks lifted + + EXPECT_FALSE(blocker.isCancelled()); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id)); + EXPECT_FALSE(blocker.isCancelledForPartition(partition_id2)); +} + +TEST(PartitionActionBlocker, TestAutomaticCompactPartitionBlockers) +{ + const size_t partitions_count = 100; + const std::string partition_id = "some partition id"; + PartitionActionBlocker blocker; + + for (size_t i = 0; i < partitions_count; ++i) + { + blocker.cancelForPartition(partition_id + "_" + std::to_string(i)); + } + + // Automatic cleanup happens once in a while, 100 stale locks should trigger it. + EXPECT_LT(blocker.countPartitionBlockers(), partitions_count); +} + +TEST(PartitionActionBlocker, TestCompactPartitionBlockers) +{ + const size_t partitions_count = 100; + const std::string partition_id = "some partition id"; + PartitionActionBlocker blocker; + + for (size_t i = 0; i < partitions_count; ++i) + { + blocker.cancelForPartition(partition_id + "_" + std::to_string(i)); + } + // Manually cleanup all stale blockers (all blockers in this case). + blocker.compactPartitionBlockers(); + + EXPECT_EQ(0, blocker.countPartitionBlockers()); +} + +TEST(PartitionActionBlocker, TestCompactPartitionBlockersDoesntRemoveActiveBlockers) +{ + const size_t partitions_count = 100; + const std::string partition_id = "some partition id"; + PartitionActionBlocker blocker; + + auto lock_foo = blocker.cancelForPartition("FOO"); + for (size_t i = 0; i < partitions_count; ++i) + { + blocker.cancelForPartition(partition_id + "_" + std::to_string(i)); + } + auto lock_bar = blocker.cancelForPartition("BAR"); + + EXPECT_LT(2, blocker.countPartitionBlockers()); + + // Manually cleanup all stale blockers (all except held by lock_foo and lock_bar). + blocker.compactPartitionBlockers(); + + EXPECT_EQ(2, blocker.countPartitionBlockers()); +} + +TEST(PartitionActionBlocker, TestFormatDebug) +{ + // Do not validate contents, just make sure that something is printed out + + const size_t partitions_count = 100; + const std::string partition_id = "some partition id"; + PartitionActionBlocker blocker; + + auto global_lock = blocker.cancel(); + auto lock_foo = blocker.cancelForPartition("FOO"); + auto lock_foo2 = blocker.cancelForPartition("FOO"); + for (size_t i = 0; i < partitions_count; ++i) + { + blocker.cancelForPartition(partition_id + "_" + std::to_string(i)); + } + auto lock_bar = blocker.cancelForPartition("BAR"); + + EXPECT_NE("", blocker.formatDebug()); +} diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index a17fd163253..6dff60aeaa9 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -144,7 +144,7 @@ private: bool supports_subset_of_columns, ContextPtr local_context) override { - auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, supports_subset_of_columns); + auto info = DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns); if (!current_metadata) { Storage::updateConfiguration(local_context); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index bc27820707c..040ce8db51d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -247,9 +247,9 @@ ReadFromFormatInfo StorageObjectStorage::prepareReadingFromFormat( const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, bool supports_subset_of_columns, - ContextPtr /* local_context */) + ContextPtr local_context) { - return DB::prepareReadingFromFormat(requested_columns, storage_snapshot, supports_subset_of_columns); + return DB::prepareReadingFromFormat(requested_columns, storage_snapshot, local_context, supports_subset_of_columns); } void StorageObjectStorage::read( diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index ded4babdb46..afa9a01666b 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -380,6 +380,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade compression_method, need_only_count); + input_format->setSerializationHints(read_from_format_info.serialization_hints); + if (key_condition_) input_format->setKeyCondition(key_condition_); diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 2fdbce15503..17cd1b5ac1f 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -298,7 +298,7 @@ void StorageObjectStorageQueue::read( } auto this_ptr = std::static_pointer_cast(shared_from_this()); - auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); + auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, local_context, supportsSubsetOfColumns(local_context)); auto reading = std::make_unique( column_names, @@ -459,6 +459,7 @@ bool StorageObjectStorageQueue::streamToViews() auto read_from_format_info = prepareReadingFromFormat( block_io.pipeline.getHeader().getNames(), storage_snapshot, + queue_context, supportsSubsetOfColumns(queue_context)); Pipes pipes; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 5da4c83f77b..2d09e4fe53f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -23,15 +23,15 @@ #include -#include +#include #include #include #include -#include +#include #include #include -#include -#include +#include +#include #include #include @@ -40,8 +40,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -1003,7 +1004,8 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu String new_query_str; { WriteBufferFromOwnString buf; - IAST::FormatSettings ast_format_settings(buf, /*one_line*/ true, /*hilite*/ false, /*always_quote_identifiers_=*/ true); + IAST::FormatSettings ast_format_settings( + /*ostr_=*/buf, /*one_line*/ true, /*hilite*/ false, /*identifier_quoting_rule_=*/IdentifierQuotingRule::Always); new_query->IAST::format(ast_format_settings); new_query_str = buf.str(); } @@ -1123,7 +1125,8 @@ std::optional StorageDistributed::distributedWriteFromClusterStor String new_query_str; { WriteBufferFromOwnString buf; - IAST::FormatSettings ast_format_settings(buf, /*one_line*/ true, /*hilite*/ false, /*always_quote_identifiers*/ true); + IAST::FormatSettings ast_format_settings( + /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); new_query->IAST::format(ast_format_settings); new_query_str = buf.str(); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 1d846b6bb0f..46f4800b497 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -99,6 +99,7 @@ namespace Setting extern const SettingsLocalFSReadMethod storage_file_read_method; extern const SettingsBool use_cache_for_count_from_files; extern const SettingsInt64 zstd_window_log_max; + extern const SettingsBool enable_parsing_to_custom_serialization; } namespace ErrorCodes @@ -1136,7 +1137,6 @@ void StorageFile::setStorageMetadata(CommonArguments args) setInMemoryMetadata(storage_metadata); } - static std::chrono::seconds getLockTimeout(const ContextPtr & context) { const Settings & settings = context->getSettingsRef(); @@ -1209,6 +1209,7 @@ StorageFileSource::StorageFileSource( , requested_columns(info.requested_columns) , requested_virtual_columns(info.requested_virtual_columns) , block_for_format(info.format_header) + , serialization_hints(info.serialization_hints) , max_block_size(max_block_size_) , need_only_count(need_only_count_) { @@ -1439,6 +1440,8 @@ Chunk StorageFileSource::generate() storage->format_name, *read_buf, block_for_format, getContext(), max_block_size, storage->format_settings, max_parsing_threads, std::nullopt, /*is_remote_fs*/ false, CompressionMethod::None, need_only_count); + input_format->setSerializationHints(serialization_hints); + if (key_condition) input_format->setKeyCondition(key_condition); @@ -1630,7 +1633,7 @@ void StorageFile::read( auto this_ptr = std::static_pointer_cast(shared_from_this()); - auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(context)); + auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, context, supportsSubsetOfColumns(context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && context->getSettingsRef()[Setting::optimize_count_from_files]; diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index bb969c1877c..6b21353f161 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -296,6 +296,7 @@ private: NamesAndTypesList requested_columns; NamesAndTypesList requested_virtual_columns; Block block_for_format; + SerializationInfoByName serialization_hints; UInt64 max_block_size; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 552e3ab4bc0..43a8ccf969c 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -694,7 +694,7 @@ void StorageLog::loadMarks(const WriteLock & lock /* already locked exclusively for (auto & data_file : data_files) data_file.marks.resize(num_marks); - std::unique_ptr marks_rb = disk->readFile(marks_file_path, ReadSettings().adjustBufferSize(32768)); + std::unique_ptr marks_rb = disk->readFile(marks_file_path, getReadSettings().adjustBufferSize(32768)); for (size_t i = 0; i != num_marks; ++i) { for (auto & data_file : data_files) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c32d9a2c213..80d646205bf 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -220,6 +220,7 @@ void StorageMerge::forEachTable(F && func) const getFirstTable([&func](const auto & table) { func(table); + /// Always continue to the next table. return false; }); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c921adaf2fd..f4d2ee67bb6 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -400,17 +400,18 @@ void StorageMergeTree::alter( DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata); + { + /// Reset Object columns, because column of type + /// Object may be added or dropped by alter. + auto parts_lock = lockParts(); + resetObjectColumnsFromActiveParts(parts_lock); + resetSerializationHints(parts_lock); + } + if (!maybe_mutation_commands.empty()) mutation_version = startMutation(maybe_mutation_commands, local_context); } - { - /// Reset Object columns, because column of type - /// Object may be added or dropped by alter. - auto parts_lock = lockParts(); - resetObjectColumnsFromActiveParts(parts_lock); - } - if (!maybe_mutation_commands.empty() && query_settings[Setting::alter_sync] > 0) waitForMutation(mutation_version, false); } @@ -1161,7 +1162,7 @@ bool StorageMergeTree::merge( { std::unique_lock lock(currently_processing_in_background_mutex); - if (merger_mutator.merges_blocker.isCancelled()) + if (merger_mutator.merges_blocker.isCancelledForPartition(partition_id)) throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); merge_mutate_entry = selectPartsToMerge( @@ -1429,8 +1430,19 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign has_mutations = !current_mutations_by_version.empty(); } + auto isCancelled = [&merges_blocker = merger_mutator.merges_blocker](const MergeMutateSelectedEntryPtr & entry) + { + if (entry->future_part) + return merges_blocker.isCancelledForPartition(entry->future_part->part_info.partition_id); + + return merges_blocker.isCancelled(); + }; + if (merge_entry) { + if (isCancelled(merge_entry)) + return false; + auto task = std::make_shared(*this, metadata_snapshot, /* deduplicate */ false, Names{}, /* cleanup */ false, merge_entry, shared_lock, common_assignee_trigger); task->setCurrentTransaction(std::move(transaction_for_merge), std::move(txn)); bool scheduled = assignee.scheduleMergeMutateTask(task); @@ -1442,6 +1454,9 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign } if (mutate_entry) { + if (isCancelled(mutate_entry)) + return false; + /// We take new metadata snapshot here. It's because mutation commands can be executed only with metadata snapshot /// which is equal or more fresh than commands themselves. In extremely rare case it can happen that we will have alter /// in between we took snapshot above and selected commands. That is why we take new snapshot here. @@ -1661,6 +1676,66 @@ bool StorageMergeTree::optimize( return true; } +namespace +{ + +size_t countOccurrences(const StorageMergeTree::DataParts & haystack, const DataPartsVector & needle) +{ + size_t total = 0; + for (const auto & n : needle) + total += haystack.count(n); + + return total; +} + +auto getNameWithState(const auto & parts) +{ + return std::views::transform(parts, [](const auto & p) + { + return p->getNameWithState(); + }); +} + +} + +// Same as stopMergesAndWait, but waits only for merges on parts belonging to a certain partition. +ActionLock StorageMergeTree::stopMergesAndWaitForPartition(String partition_id) +{ + LOG_DEBUG(log, "StorageMergeTree::stopMergesAndWaitForPartition partition_id: `{}`", partition_id); + /// Stop all merges and prevent new from starting, BUT unlike stopMergesAndWait(), only wait for the merges on small set of parts to finish. + + std::unique_lock lock(currently_processing_in_background_mutex); + + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = merger_mutator.merges_blocker.cancelForPartition(partition_id); + + const DataPartsVector parts_to_wait = getDataPartsVectorInPartitionForInternalUsage(MergeTreeDataPartState::Active, partition_id); + LOG_TRACE(log, "StorageMergeTree::stopMergesAndWaitForPartition parts to wait: {} ({} items)", + fmt::join(getNameWithState(parts_to_wait), ", "), parts_to_wait.size()); + + LOG_DEBUG(log, "StorageMergeTree::stopMergesAndWaitForPartition all mutating parts: {} ({} items)", + fmt::join(getNameWithState(currently_merging_mutating_parts), ", "), currently_merging_mutating_parts.size()); + + // TODO allow to stop merges in specific partition only (like it's done in ReplicatedMergeTree) + + while (size_t still_merging = countOccurrences(currently_merging_mutating_parts, parts_to_wait)) + { + LOG_DEBUG(log, "StorageMergeTree::stopMergesAndWaitForPartition Waiting for currently running merges ({} {} parts are merging right now)", + fmt::join(getNameWithState(currently_merging_mutating_parts), ", "), still_merging); + + if (std::cv_status::timeout == currently_processing_in_background_condition.wait_for( + lock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC))) + { + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout while waiting for already running merges"); + } + } + + LOG_DEBUG(log, "StorageMergeTree::stopMergesAndWaitForPartition done waiting, still merging {} ({} items)", + fmt::join(getNameWithState(currently_merging_mutating_parts), ", "), currently_merging_mutating_parts.size()); + return merge_blocker; +} + ActionLock StorageMergeTree::stopMergesAndWait() { /// TODO allow to stop merges in specific partition only (like it's done in ReplicatedMergeTree) @@ -2095,10 +2170,27 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition( void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr local_context) { assertNotReadonly(); + LOG_DEBUG(log, "StorageMergeTree::replacePartitionFrom\tsource_table: {}, replace: {}", source_table->getStorageID().getShortName(), replace); auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto lock2 = source_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); - auto merges_blocker = stopMergesAndWait(); + + bool is_all = partition->as()->all; + + String partition_id; + + if (is_all) + { + if (replace) + throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH PARTITION ALL currently"); + auto merges_blocker = stopMergesAndWait(); + } + else + { + partition_id = getPartitionIDFromQuery(partition, local_context); + auto merges_blocker = stopMergesAndWaitForPartition(partition_id); + } + auto source_metadata_snapshot = source_table->getInMemoryMetadataPtr(); auto my_metadata_snapshot = getInMemoryMetadataPtr(); @@ -2107,20 +2199,11 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con MergeTreeData & src_data = checkStructureAndGetMergeTreeData(source_table, source_metadata_snapshot, my_metadata_snapshot); DataPartsVector src_parts; - String partition_id; - bool is_all = partition->as()->all; - if (is_all) - { - if (replace) - throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH PARTITION ALL currently"); + if (is_all) src_parts = src_data.getVisibleDataPartsVector(local_context); - } else - { - partition_id = getPartitionIDFromQuery(partition, local_context); src_parts = src_data.getVisibleDataPartsVectorInPartition(local_context, partition_id); - } MutableDataPartsVector dst_parts; std::vector dst_parts_locks; diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index ef333fe3f18..7bc070b12b4 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -189,6 +189,7 @@ private: /// If not force, then take merges selector and check that part is not participating in background operations. MergeTreeDataPartPtr outdatePart(MergeTreeTransaction * txn, const String & part_name, bool force, bool clear_without_timeout = true); ActionLock stopMergesAndWait(); + ActionLock stopMergesAndWaitForPartition(String partition_id); /// Allocate block number for new mutation, write mutation to disk /// and into in-memory structures. Wake up merge-mutation task. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 30979da55a2..a3d529c5fbb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5724,7 +5724,8 @@ std::optional StorageReplicatedMergeTree::distributedWriteFromClu String query_str; { WriteBufferFromOwnString buf; - IAST::FormatSettings ast_format_settings(buf, /*one_line*/ true, /*hilite*/ false, /*always_quote_identifiers*/ true); + IAST::FormatSettings ast_format_settings( + /*ostr_=*/buf, /*one_line=*/true, /*hilite=*/false, /*identifier_quoting_rule=*/IdentifierQuotingRule::Always); query.IAST::format(ast_format_settings); query_str = buf.str(); } @@ -6075,6 +6076,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// Object may be added or dropped by alter. auto parts_lock = lockParts(); resetObjectColumnsFromActiveParts(parts_lock); + resetSerializationHints(parts_lock); } return true; diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 5692da4e454..990a2019cc9 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -280,7 +280,7 @@ void StorageSetOrJoinBase::restore() void StorageSetOrJoinBase::restoreFromFile(const String & file_path) { ContextPtr ctx = nullptr; - auto backup_buf = disk->readFile(file_path); + auto backup_buf = disk->readFile(file_path, getReadSettings()); CompressedReadBuffer compressed_backup_buf(*backup_buf); NativeReader backup_stream(compressed_backup_buf, 0); diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index ddc39cfb110..6fd8bfe5b6b 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -478,7 +478,7 @@ void StorageStripeLog::loadIndices(const WriteLock & lock /* already locked excl if (disk->exists(index_file_path)) { - CompressedReadBufferFromFile index_in(disk->readFile(index_file_path, ReadSettings{}.adjustBufferSize(4096))); + CompressedReadBufferFromFile index_in(disk->readFile(index_file_path, getContext()->getReadSettings().adjustBufferSize(4096))); indices.read(index_in); } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 42648ad73e6..80c07658055 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -408,6 +408,8 @@ StorageURLSource::StorageURLSource( compression_method, need_only_count); + input_format->setSerializationHints(info.serialization_hints); + if (key_condition) input_format->setKeyCondition(key_condition); @@ -1127,7 +1129,7 @@ void IStorageURLBase::read( size_t num_streams) { auto params = getReadURIParams(column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size); - auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); + auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, local_context, supportsSubsetOfColumns(local_context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && local_context->getSettingsRef()[Setting::optimize_count_from_files]; @@ -1297,7 +1299,7 @@ void StorageURLWithFailover::read( size_t num_streams) { auto params = getReadURIParams(column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size); - auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); + auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, local_context, supportsSubsetOfColumns(local_context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) && local_context->getSettingsRef()[Setting::optimize_count_from_files]; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index cb438e0efa6..bcbcd4f66c8 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -187,7 +187,7 @@ void StorageView::read( /// It's expected that the columns read from storage are not constant. /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. ActionsDAG materializing_actions(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); - materializing_actions.addMaterializingOutputActions(); + materializing_actions.addMaterializingOutputActions(/*materialize_sparse=*/ true); auto materializing = std::make_unique(query_plan.getCurrentDataStream(), std::move(materializing_actions)); materializing->setStepDescription("Materialize constants after VIEW subquery"); diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 710ee503086..47ca3ad89a8 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -34,6 +34,9 @@ StorageSystemColumns::StorageSystemColumns(const StorageID & table_id_) : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; + + /// NOTE: when changing the list of columns, take care of the ColumnsSource::generate method, + /// when they are referenced by their numeric positions. storage_metadata.setColumns(ColumnsDescription( { { "database", std::make_shared(), "Database name."}, @@ -126,9 +129,7 @@ protected: { StoragePtr storage = storages.at(std::make_pair(database_name, table_name)); - TableLockHolder table_lock; - - table_lock = storage->tryLockForShare(query_id, lock_acquire_timeout); + TableLockHolder table_lock = storage->tryLockForShare(query_id, lock_acquire_timeout); if (table_lock == nullptr) { @@ -139,11 +140,18 @@ protected: auto metadata_snapshot = storage->getInMemoryMetadataPtr(); columns = metadata_snapshot->getColumns(); - cols_required_for_partition_key = metadata_snapshot->getColumnsRequiredForPartitionKey(); - cols_required_for_sorting_key = metadata_snapshot->getColumnsRequiredForSortingKey(); - cols_required_for_primary_key = metadata_snapshot->getColumnsRequiredForPrimaryKey(); - cols_required_for_sampling = metadata_snapshot->getColumnsRequiredForSampling(); - column_sizes = storage->getColumnSizes(); + /// Certain information about a table - should be calculated only when the corresponding columns are queried. + if (columns_mask[7] || columns_mask[8] || columns_mask[9]) + column_sizes = storage->getColumnSizes(); + + if (columns_mask[11]) + cols_required_for_partition_key = metadata_snapshot->getColumnsRequiredForPartitionKey(); + if (columns_mask[12]) + cols_required_for_sorting_key = metadata_snapshot->getColumnsRequiredForSortingKey(); + if (columns_mask[13]) + cols_required_for_primary_key = metadata_snapshot->getColumnsRequiredForPrimaryKey(); + if (columns_mask[14]) + cols_required_for_sampling = metadata_snapshot->getColumnsRequiredForSampling(); } /// A shortcut: if we don't allow to list this table in SHOW TABLES, also exclude it from system.columns. diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index 641bbb319d5..ddd86f8736b 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -150,10 +150,10 @@ void StorageSystemQuotas::backupData( } void StorageSystemQuotas::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restoreFromBackup(restorer); + access_control.restoreFromBackup(restorer, data_path_in_backup); } } diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp index 9bfddc25ebf..2409580bbd2 100644 --- a/src/Storages/System/StorageSystemRoles.cpp +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -70,10 +70,10 @@ void StorageSystemRoles::backupData( } void StorageSystemRoles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restoreFromBackup(restorer); + access_control.restoreFromBackup(restorer, data_path_in_backup); } } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 93c5ba60a7f..759535e33b7 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -160,10 +160,10 @@ void StorageSystemRowPolicies::backupData( } void StorageSystemRowPolicies::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restoreFromBackup(restorer); + access_control.restoreFromBackup(restorer, data_path_in_backup); } } diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp index 795152e31f3..7aa9fae9ebc 100644 --- a/src/Storages/System/StorageSystemSettingsProfiles.cpp +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -101,10 +101,10 @@ void StorageSystemSettingsProfiles::backupData( } void StorageSystemSettingsProfiles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restoreFromBackup(restorer); + access_control.restoreFromBackup(restorer, data_path_in_backup); } } diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 135a853777f..a828b1e189f 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -57,8 +57,10 @@ bool needTable(const DatabasePtr & database, const Block & header) } return false; } + } + namespace detail { ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context) diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index ce4950f5e7b..7589afdeb3e 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -261,10 +261,10 @@ void StorageSystemUsers::backupData( } void StorageSystemUsers::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restoreFromBackup(restorer); + access_control.restoreFromBackup(restorer, data_path_in_backup); } } diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp index 72a7ba38429..9a83e518058 100644 --- a/src/Storages/System/StorageSystemZooKeeperConnection.cpp +++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp @@ -34,7 +34,7 @@ ColumnsDescription StorageSystemZooKeeperConnection::getColumnsDescription() /* 6 */ {"is_expired", std::make_shared(), "Is the current connection expired."}, /* 7 */ {"keeper_api_version", std::make_shared(), "Keeper API version."}, /* 8 */ {"client_id", std::make_shared(), "Session id of the connection."}, - /* 9 */ {"xid", std::make_shared(), "XID of the current session."}, + /* 9 */ {"xid", std::make_shared(), "XID of the current session."}, /* 10*/ {"enabled_feature_flags", std::make_shared(std::move(feature_flags_enum)), "Feature flags which are enabled. Only applicable to ClickHouse Keeper." }, diff --git a/src/Storages/prepareReadingFromFormat.cpp b/src/Storages/prepareReadingFromFormat.cpp index 406b7f379f9..b87af449dc5 100644 --- a/src/Storages/prepareReadingFromFormat.cpp +++ b/src/Storages/prepareReadingFromFormat.cpp @@ -1,10 +1,19 @@ #include #include +#include +#include +#include +#include namespace DB { -ReadFromFormatInfo prepareReadingFromFormat(const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, bool supports_subset_of_columns) +namespace Setting +{ + extern const SettingsBool enable_parsing_to_custom_serialization; +} + +ReadFromFormatInfo prepareReadingFromFormat(const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool supports_subset_of_columns) { ReadFromFormatInfo info; /// Collect requested virtual columns and remove them from requested columns. @@ -72,7 +81,35 @@ ReadFromFormatInfo prepareReadingFromFormat(const Strings & requested_columns, c /// Create header for InputFormat with columns that will be read from the data. info.format_header = storage_snapshot->getSampleBlockForColumns(info.columns_description.getNamesOfPhysical()); + info.serialization_hints = getSerializationHintsForFileLikeStorage(storage_snapshot->metadata, context); return info; } +SerializationInfoByName getSerializationHintsForFileLikeStorage(const StorageMetadataPtr & metadata_snapshot, const ContextPtr & context) +{ + if (!context->getSettingsRef()[Setting::enable_parsing_to_custom_serialization]) + return {}; + + auto insertion_table = context->getInsertionTable(); + if (!insertion_table) + return {}; + + auto storage_ptr = DatabaseCatalog::instance().tryGetTable(insertion_table, context); + if (!storage_ptr) + return {}; + + const auto & our_columns = metadata_snapshot->getColumns(); + const auto & storage_columns = storage_ptr->getInMemoryMetadataPtr()->getColumns(); + auto storage_hints = storage_ptr->getSerializationHints(); + SerializationInfoByName res; + + for (const auto & hint : storage_hints) + { + if (our_columns.tryGetPhysical(hint.first) == storage_columns.tryGetPhysical(hint.first)) + res.insert(hint); + } + + return res; +} + } diff --git a/src/Storages/prepareReadingFromFormat.h b/src/Storages/prepareReadingFromFormat.h index e4d62c29ec6..02e42056d0c 100644 --- a/src/Storages/prepareReadingFromFormat.h +++ b/src/Storages/prepareReadingFromFormat.h @@ -1,6 +1,8 @@ #pragma once #include #include +#include +#include namespace DB { @@ -19,8 +21,14 @@ namespace DB NamesAndTypesList requested_columns; /// The list of requested virtual columns. NamesAndTypesList requested_virtual_columns; + /// Hints for the serialization of columns. + /// For example can be retrieved from the destination table in INSERT SELECT query. + SerializationInfoByName serialization_hints; }; /// Get all needed information for reading from data in some input format. - ReadFromFormatInfo prepareReadingFromFormat(const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, bool supports_subset_of_columns); + ReadFromFormatInfo prepareReadingFromFormat(const Strings & requested_columns, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool supports_subset_of_columns); + + /// Returns the serialization hints from the insertion table (if it's set in the Context). + SerializationInfoByName getSerializationHintsForFileLikeStorage(const StorageMetadataPtr & metadata_snapshot, const ContextPtr & context); } diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 0cf4f706b9c..fa98dfe036f 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -386,13 +386,16 @@ String transformQueryForExternalDatabaseImpl( ASTPtr select_ptr = select; dropAliases(select_ptr); - + IdentifierQuotingRule identifier_quoting_rule = IdentifierQuotingRule::Always; WriteBufferFromOwnString out; IAST::FormatSettings settings( - out, /*one_line*/ true, /*hilite*/ false, - /*always_quote_identifiers*/ identifier_quoting_style != IdentifierQuotingStyle::None, - /*identifier_quoting_style*/ identifier_quoting_style, /*show_secrets_*/ true, - /*literal_escaping_style*/ literal_escaping_style); + /*ostr_=*/out, + /*one_line=*/true, + /*hilite=*/false, + /*identifier_quoting_rule=*/identifier_quoting_rule, + /*identifier_quoting_style=*/identifier_quoting_style, + /*show_secrets_=*/true, + /*literal_escaping_style=*/literal_escaping_style); select->format(settings); diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 8bc0f51dfc7..eec70f6e60e 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -7,6 +7,7 @@ import sys from pathlib import Path from build_download_helper import read_build_urls +from ci_config import CI from clickhouse_helper import CiLogsCredentials from docker_images_helper import DockerImage, get_docker_image, pull_image from env_helper import REPORT_PATH, TEMP_PATH @@ -14,7 +15,6 @@ from pr_info import PRInfo from report import FAIL, FAILURE, OK, SUCCESS, JobReport, TestResult from stopwatch import Stopwatch from tee_popen import TeePopen -from ci_config import CI IMAGE_NAME = "clickhouse/fuzzer" diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index 03669218d29..a796f63de6c 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -34,11 +34,11 @@ from typing import List, Optional import __main__ +from ci_config import Labels from env_helper import TEMP_PATH from get_robot_token import get_best_robot_token from git_helper import GIT_PREFIX, git_runner, is_shallow from github_helper import GitHub, PullRequest, PullRequests, Repository -from ci_config import Labels from ssh import SSHKey diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index 07f318207a4..d3c2ec60b96 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -1,15 +1,15 @@ import argparse import json import os -from typing import Union, Dict, List +from typing import Dict, List, Union import boto3 import requests from botocore.exceptions import ClientError -from pr_info import PRInfo from ci_config import CI from ci_utils import WithIter +from pr_info import PRInfo class Channels(metaclass=WithIter): diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 34201748741..c62727e4e2b 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -1,7 +1,7 @@ import copy from dataclasses import dataclass, field from pathlib import Path -from typing import List, Union, Iterable, Optional, Literal, Any +from typing import Any, Iterable, List, Literal, Optional, Union from ci_utils import WithIter from integration_test_images import IMAGES diff --git a/tests/ci/ci_metadata.py b/tests/ci/ci_metadata.py index 67106262634..ce0358f5e76 100644 --- a/tests/ci/ci_metadata.py +++ b/tests/ci/ci_metadata.py @@ -1,18 +1,17 @@ from pathlib import Path from typing import Optional +from ci_utils import GH from env_helper import ( - S3_BUILDS_BUCKET, - TEMP_PATH, - GITHUB_UPSTREAM_REPOSITORY, GITHUB_REPOSITORY, + GITHUB_UPSTREAM_REPOSITORY, + S3_BUILDS_BUCKET, S3_BUILDS_BUCKET_PUBLIC, + TEMP_PATH, ) from s3_helper import S3Helper -from ci_utils import GH from synchronizer_utils import SYNC_BRANCH_PREFIX - # pylint: disable=too-many-lines diff --git a/tests/ci/clickbench.py b/tests/ci/clickbench.py index a1988abb1f5..349a848e9c7 100644 --- a/tests/ci/clickbench.py +++ b/tests/ci/clickbench.py @@ -10,15 +10,13 @@ from pathlib import Path from typing import List, Tuple from build_download_helper import download_all_deb_packages -from clickhouse_helper import ( - CiLogsCredentials, -) -from docker_images_helper import get_docker_image, pull_image, DockerImage -from env_helper import TEMP_PATH, REPORT_PATH +from clickhouse_helper import CiLogsCredentials +from docker_images_helper import DockerImage, get_docker_image, pull_image +from env_helper import REPORT_PATH, TEMP_PATH from pr_info import PRInfo +from report import ERROR, SUCCESS, JobReport, StatusType, TestResults from stopwatch import Stopwatch from tee_popen import TeePopen -from report import ERROR, SUCCESS, JobReport, StatusType, TestResults def get_image_name() -> str: diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 0725f7100d1..99f4778fe8f 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -2,8 +2,8 @@ import fileinput import json import logging -import time import os +import time from pathlib import Path from typing import Any, Dict, List, Optional diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py index f49c872b5a0..74c6a6add4a 100644 --- a/tests/ci/compress_files.py +++ b/tests/ci/compress_files.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -import subprocess import logging - +import subprocess from pathlib import Path from typing import Optional - PIGZ = Path("/usr/bin/pigz") SUFFIX = ".zst" diff --git a/tests/ci/digest_helper.py b/tests/ci/digest_helper.py index 4dcfb03c04f..bb08f2c2bbd 100644 --- a/tests/ci/digest_helper.py +++ b/tests/ci/digest_helper.py @@ -5,21 +5,21 @@ from dataclasses import asdict from hashlib import md5 from logging import getLogger from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union from sys import modules +from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union -from docker_images_helper import get_images_info -from git_helper import Runner -from env_helper import ROOT_DIR -from ci_utils import cd from ci_config import CI +from ci_utils import cd +from docker_images_helper import get_images_info +from env_helper import ROOT_DIR +from git_helper import Runner DOCKER_DIGEST_LEN = 12 JOB_DIGEST_LEN = 10 if TYPE_CHECKING: - from hashlib import ( # pylint:disable=no-name-in-module,ungrouped-imports - _Hash as HASH, + from hashlib import ( + _Hash as HASH, # pylint:disable=no-name-in-module,ungrouped-imports ) else: HASH = "_Hash" diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 998a97273ae..274ed31d984 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -10,9 +10,8 @@ from typing import Any, List import boto3 # type: ignore -from build_download_helper import ( - read_build_urls, -) +from build_download_helper import read_build_urls +from ci_config import CI from compress_files import compress_fast from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH from get_robot_token import get_parameter_from_ssm @@ -21,7 +20,6 @@ from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults from ssh import SSHKey from stopwatch import Stopwatch from tee_popen import TeePopen -from ci_config import CI JEPSEN_GROUP_NAME = "jepsen_group" diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py index ddeb76adf7e..0a798ebf718 100644 --- a/tests/ci/merge_pr.py +++ b/tests/ci/merge_pr.py @@ -14,19 +14,19 @@ from github.PaginatedList import PaginatedList from github.PullRequestReview import PullRequestReview from github.WorkflowRun import WorkflowRun +from ci_config import CI from commit_status_helper import ( - get_commit_filtered_statuses, get_commit, + get_commit_filtered_statuses, trigger_mergeable_check, update_upstream_sync_status, ) +from env_helper import GITHUB_REPOSITORY, GITHUB_UPSTREAM_REPOSITORY from get_robot_token import get_best_robot_token from github_helper import GitHub, NamedUser, PullRequest, Repository from pr_info import PRInfo from report import SUCCESS -from env_helper import GITHUB_UPSTREAM_REPOSITORY, GITHUB_REPOSITORY from synchronizer_utils import SYNC_BRANCH_PREFIX -from ci_config import CI # The team name for accepted approvals TEAM_NAME = getenv("GITHUB_TEAM_NAME", "core") diff --git a/tests/ci/release.py b/tests/ci/release.py index b26d6205f3b..ed9d60a5cad 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -24,8 +24,8 @@ import subprocess from contextlib import contextmanager from typing import Any, Final, Iterator, List, Optional, Tuple -from git_helper import Git, commit, release_branch from ci_config import Labels +from git_helper import Git, commit, release_branch from report import SUCCESS from version_helper import ( FILE_WITH_VERSION_PATH, diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 55a0c383812..2d3ac4f1485 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -6,6 +6,7 @@ from typing import Tuple from github import Github +from ci_config import CI from commit_status_helper import ( create_ci_report, format_description, @@ -16,11 +17,9 @@ from commit_status_helper import ( ) from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL from get_robot_token import get_best_robot_token -from ci_config import CI from pr_info import PRInfo from report import FAILURE, PENDING, SUCCESS, StatusType - TRUSTED_ORG_IDS = { 54801242, # clickhouse } diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 86656e6e7c0..9a40ad1277f 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -9,6 +9,7 @@ from typing import Any, List, Union import boto3 # type: ignore import botocore # type: ignore + from compress_files import compress_file_fast from env_helper import ( IS_CI, diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index a68db8b9791..e03a44bf56a 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -7,12 +7,12 @@ import sys from pathlib import Path from build_download_helper import read_build_urls +from ci_config import CI from docker_images_helper import DockerImage, get_docker_image, pull_image from env_helper import REPORT_PATH, TEMP_PATH from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults from stopwatch import Stopwatch from tee_popen import TeePopen -from ci_config import CI IMAGE_NAME = "clickhouse/sqlancer-test" diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py index 8e6ca6ff87f..34a204746ae 100644 --- a/tests/ci/sqltest.py +++ b/tests/ci/sqltest.py @@ -7,12 +7,12 @@ import sys from pathlib import Path from build_download_helper import read_build_urls +from ci_config import CI from docker_images_helper import get_docker_image, pull_image from env_helper import REPORT_PATH, TEMP_PATH from pr_info import PRInfo from report import SUCCESS, JobReport, TestResult from stopwatch import Stopwatch -from ci_config import CI IMAGE_NAME = "clickhouse/sqltest" diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index c15a79a345e..82e8677f9a0 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -214,13 +214,13 @@ def main(): state, description, test_results, additional_files = process_result(temp_path) autofix_description = "" - fail_cnt = 0 + push_fix = args.push for result in test_results: - if result.status in (FAILURE, FAIL): - # do not autofix if not only black failed - fail_cnt += 1 + if result.status in (FAILURE, FAIL) and push_fix: + # do not autofix if something besides isort and black is failed + push_fix = any(result.name.endswith(check) for check in ("isort", "black")) - if args.push and fail_cnt == 1: + if push_fix: try: commit_push_staged(pr_info) except subprocess.SubprocessError: diff --git a/tests/ci/sync_pr.py b/tests/ci/sync_pr.py index 1b71231f820..2003eab65c4 100644 --- a/tests/ci/sync_pr.py +++ b/tests/ci/sync_pr.py @@ -5,12 +5,12 @@ import argparse import sys +from ci_config import CI from commit_status_helper import get_commit, post_commit_status from get_robot_token import get_best_robot_token from github_helper import GitHub from pr_info import PRInfo from report import SUCCESS -from ci_config import CI def parse_args() -> argparse.Namespace: diff --git a/tests/ci/test_ci_cache.py b/tests/ci/test_ci_cache.py index 81d649b246b..5d82ab96d14 100644 --- a/tests/ci/test_ci_cache.py +++ b/tests/ci/test_ci_cache.py @@ -1,16 +1,17 @@ #!/usr/bin/env python +import shutil +import unittest from hashlib import md5 from pathlib import Path -import shutil from typing import Dict, Set -import unittest -from s3_helper import S3Helper + from ci_cache import CiCache -from digest_helper import JOB_DIGEST_LEN -from commit_status_helper import CommitStatusData -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH from ci_config import CI +from commit_status_helper import CommitStatusData +from digest_helper import JOB_DIGEST_LEN +from env_helper import S3_BUILDS_BUCKET, TEMP_PATH +from s3_helper import S3Helper def _create_mock_digest_1(string): diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py index c3e55aeac06..c6689f102ea 100644 --- a/tests/ci/test_ci_config.py +++ b/tests/ci/test_ci_config.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 import copy -import unittest import random +import unittest -from ci_config import CI import ci as CIPY -from ci_settings import CiSettings -from pr_info import PRInfo, EventType -from s3_helper import S3Helper from ci_cache import CiCache +from ci_config import CI +from ci_settings import CiSettings from ci_utils import Utils - +from pr_info import EventType, PRInfo +from s3_helper import S3Helper _TEST_EVENT_JSON = {"dummy": "dummy"} diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index 534885dda2b..7889072df35 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -3,8 +3,9 @@ # type: ignore import unittest -from ci_settings import CiSettings + from ci_config import CI +from ci_settings import CiSettings _TEST_BODY_1 = """ #### Run only: diff --git a/tests/ci/test_docker.py b/tests/ci/test_docker.py index 58ebe4ecbb1..cd1ed1a9118 100644 --- a/tests/ci/test_docker.py +++ b/tests/ci/test_docker.py @@ -2,8 +2,8 @@ import unittest -from version_helper import get_version_from_string import docker_server as ds +from version_helper import get_version_from_string class TestDockerServer(unittest.TestCase): diff --git a/tests/config/config.d/zookeeper.xml b/tests/config/config.d/zookeeper.xml index ce402f4850b..badba37fa95 100644 --- a/tests/config/config.d/zookeeper.xml +++ b/tests/config/config.d/zookeeper.xml @@ -3,6 +3,7 @@ random 1 + 1 127.0.0.1 9181 diff --git a/tests/config/config.d/zookeeper_fault_injection.xml b/tests/config/config.d/zookeeper_fault_injection.xml index 75b96064817..b879b9a945c 100644 --- a/tests/config/config.d/zookeeper_fault_injection.xml +++ b/tests/config/config.d/zookeeper_fault_injection.xml @@ -1,6 +1,7 @@ 1 + 1 localhost 9181 diff --git a/tests/docker_scripts/process_functional_tests_result.py b/tests/docker_scripts/process_functional_tests_result.py index 7f086313f8a..eb845ce3668 100755 --- a/tests/docker_scripts/process_functional_tests_result.py +++ b/tests/docker_scripts/process_functional_tests_result.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import os -import logging import argparse import csv +import logging +import os OK_SIGN = "[ OK " FAIL_SIGN = "[ FAIL " diff --git a/tests/docker_scripts/stateless_runner.sh b/tests/docker_scripts/stateless_runner.sh index 31889b1912d..307b41cf4f1 100755 --- a/tests/docker_scripts/stateless_runner.sh +++ b/tests/docker_scripts/stateless_runner.sh @@ -62,6 +62,8 @@ source /repo/tests/docker_scripts/utils.lib config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; then + sudo sed -i "/1<\/use_xid_64>/d" /etc/clickhouse-server/config.d/zookeeper.xml + function remove_keeper_config() { sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml diff --git a/tests/docker_scripts/stress_tests.lib b/tests/docker_scripts/stress_tests.lib index 9504581ac47..3ab52c19dbd 100644 --- a/tests/docker_scripts/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -72,8 +72,10 @@ function configure() if [[ -n "$ZOOKEEPER_FAULT_INJECTION" ]] && [[ "$ZOOKEEPER_FAULT_INJECTION" -eq 1 ]]; then randomize_config_boolean_value use_compression zookeeper_fault_injection + randomize_config_boolean_value use_xid_64 zookeeper_fault_injection else randomize_config_boolean_value use_compression zookeeper + randomize_config_boolean_value use_xid_64 zookeeper fi randomize_config_boolean_value enable_block_number_column block_number diff --git a/tests/fuzz/runner.py b/tests/fuzz/runner.py index 44259228f60..4d7ac457627 100644 --- a/tests/fuzz/runner.py +++ b/tests/fuzz/runner.py @@ -3,8 +3,8 @@ import configparser import logging import os -from pathlib import Path import subprocess +from pathlib import Path DEBUGGER = os.getenv("DEBUGGER", "") FUZZER_ARGS = os.getenv("FUZZER_ARGS", "") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index aa235118aed..88ce8640c97 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,7 +6,8 @@ import logging import os import pytest # pylint:disable=import-error; for style check -from helpers.cluster import run_and_check, is_port_free + +from helpers.cluster import is_port_free, run_and_check from helpers.network import _NetworkManager # This is a workaround for a problem with logging in pytest [1]. @@ -22,8 +23,8 @@ def pdb_history(request): Fixture loads and saves pdb history to file, so it can be preserved between runs """ if request.config.getoption("--pdb"): - import readline # pylint:disable=import-outside-toplevel import pdb # pylint:disable=import-outside-toplevel + import readline # pylint:disable=import-outside-toplevel def save_history(): readline.write_history_file(".pdb_history") diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 97405860784..1677a1536c5 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -1,7 +1,7 @@ +import logging import os import subprocess as sp import tempfile -import logging from threading import Timer DEFAULT_QUERY_TIMEOUT = 600 diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 9a75fb4321e..f5f87947c0f 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1,63 +1,64 @@ import base64 import errno -from functools import cache import http.client import logging import os -import platform -import stat import os.path as p +import platform import pprint import pwd import re +import shlex import shutil import socket +import stat import subprocess import time import traceback import urllib.parse -import shlex -import urllib3 -import requests +from functools import cache from pathlib import Path +import requests +import urllib3 + try: # Please, add modules that required for specific tests only here. # So contributors will be able to run most tests locally # without installing tons of unneeded packages that may be not so easy to install. import asyncio - from cassandra.policies import RoundRobinPolicy + import ssl + import cassandra.cluster + import nats import psycopg2 - from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import pymongo import pymysql - import nats - import ssl + from cassandra.policies import RoundRobinPolicy from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, ) + from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + from .hdfs_api import HDFSApi # imports requests_kerberos except Exception as e: logging.warning(f"Cannot import some modules, some tests may not work: {e}") +import docker from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio -from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry from helpers import pytest_xdist_logging_to_separate_files from helpers.client import QueryRuntimeException - -import docker +from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry from .client import Client +from .config_cluster import * from .random_settings import write_random_settings_config from .retry_decorator import retry -from .config_cluster import * - HELPERS_DIR = p.dirname(__file__) CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") LOCAL_DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "tests/integration/compose/") diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index 6c6435e0d78..1b2c73142f3 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import logging import os import uuid import warnings @@ -8,7 +9,6 @@ import cassandra.cluster import pymongo import pymysql.cursors import redis -import logging class ExternalSource(object): diff --git a/tests/integration/helpers/hdfs_api.py b/tests/integration/helpers/hdfs_api.py index 4e4468fef77..c85aad1083b 100644 --- a/tests/integration/helpers/hdfs_api.py +++ b/tests/integration/helpers/hdfs_api.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- -import io import gzip -import subprocess -import time -from tempfile import NamedTemporaryFile -import requests -import requests_kerberos as reqkerb -import socket -import tempfile +import io import logging import os +import socket +import subprocess +import tempfile +import time +from tempfile import NamedTemporaryFile + +import requests +import requests_kerberos as reqkerb class mk_krb_conf(object): diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index be710db37d1..bc08e7b120d 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -1,13 +1,15 @@ +import contextlib import io -import subprocess +import select import socket +import subprocess import time import typing as tp -import contextlib -import select + from kazoo.client import KazooClient -from helpers.cluster import ClickHouseCluster, ClickHouseInstance + from helpers.client import CommandRequest +from helpers.cluster import ClickHouseCluster, ClickHouseInstance def execute_keeper_client_query( diff --git a/tests/integration/helpers/mock_servers.py b/tests/integration/helpers/mock_servers.py index f2181d85e12..a7674477787 100644 --- a/tests/integration/helpers/mock_servers.py +++ b/tests/integration/helpers/mock_servers.py @@ -1,7 +1,7 @@ +import importlib import logging import os import time -import importlib # Starts simple HTTP servers written in Python. diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index f24b5924e73..153d4c2860c 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -1,9 +1,10 @@ +import ipaddress +import logging import os import subprocess import time -import logging + import docker -import ipaddress class PartitionManager: diff --git a/tests/integration/helpers/postgres_utility.py b/tests/integration/helpers/postgres_utility.py index c27a367a24e..ba6ec1fd66e 100644 --- a/tests/integration/helpers/postgres_utility.py +++ b/tests/integration/helpers/postgres_utility.py @@ -1,5 +1,6 @@ -import psycopg2 import time + +import psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT postgres_table_template = """ diff --git a/tests/integration/helpers/retry_decorator.py b/tests/integration/helpers/retry_decorator.py index e7bafbe29c1..771fab49d61 100644 --- a/tests/integration/helpers/retry_decorator.py +++ b/tests/integration/helpers/retry_decorator.py @@ -1,6 +1,6 @@ -import time import random -from typing import Type, List +import time +from typing import List, Type def retry( diff --git a/tests/integration/helpers/s3_mocks/broken_s3.py b/tests/integration/helpers/s3_mocks/broken_s3.py index 686abc76bdf..7e4dbf13618 100644 --- a/tests/integration/helpers/s3_mocks/broken_s3.py +++ b/tests/integration/helpers/s3_mocks/broken_s3.py @@ -1,15 +1,14 @@ -import logging -import sys -import threading -import random -import time -import urllib.parse import http.server +import logging +import random +import socket import socketserver import string -import socket import struct - +import sys +import threading +import time +import urllib.parse INF_COUNT = 100000000 diff --git a/tests/integration/helpers/s3_tools.py b/tests/integration/helpers/s3_tools.py index 5b727060e69..d56ffc06118 100644 --- a/tests/integration/helpers/s3_tools.py +++ b/tests/integration/helpers/s3_tools.py @@ -1,12 +1,11 @@ -from minio import Minio import glob -import os import json +import os import shutil - - from enum import Enum +from minio import Minio + class CloudUploader: diff --git a/tests/integration/helpers/s3_url_proxy_tests_util.py b/tests/integration/helpers/s3_url_proxy_tests_util.py index 9a45855acb8..775bf2db353 100644 --- a/tests/integration/helpers/s3_url_proxy_tests_util.py +++ b/tests/integration/helpers/s3_url_proxy_tests_util.py @@ -1,7 +1,6 @@ import os import time - ALL_HTTP_METHODS = {"POST", "PUT", "GET", "HEAD", "CONNECT"} diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 1c8c5c33a13..e71698692be 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -1,6 +1,6 @@ import difflib -import time import logging +import time from io import IOBase diff --git a/tests/integration/helpers/uexpect.py b/tests/integration/helpers/uexpect.py index 757a3a7f199..e6b8e4ba58d 100644 --- a/tests/integration/helpers/uexpect.py +++ b/tests/integration/helpers/uexpect.py @@ -15,9 +15,9 @@ import os import pty import re import time -from queue import Queue, Empty +from queue import Empty, Queue from subprocess import Popen -from threading import Thread, Event +from threading import Event, Thread class TimeoutError(Exception): diff --git a/tests/integration/helpers/utility.py b/tests/integration/helpers/utility.py index 0fd55569d92..82ff7196a6a 100644 --- a/tests/integration/helpers/utility.py +++ b/tests/integration/helpers/utility.py @@ -1,5 +1,5 @@ -import string import random +import string import threading diff --git a/tests/integration/helpers/wait_for_helpers.py b/tests/integration/helpers/wait_for_helpers.py index d240c71bc86..fca016f848f 100644 --- a/tests/integration/helpers/wait_for_helpers.py +++ b/tests/integration/helpers/wait_for_helpers.py @@ -1,4 +1,5 @@ import time + from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_MemoryTracking/test.py b/tests/integration/test_MemoryTracking/test.py index 517090988ee..7914bd7bad5 100644 --- a/tests/integration/test_MemoryTracking/test.py +++ b/tests/integration/test_MemoryTracking/test.py @@ -14,7 +14,9 @@ # asynchronous_metrics_update_period_s.xml. import logging + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py index f43e9e6140a..b527fcc9f14 100644 --- a/tests/integration/test_accept_invalid_certificate/test.py +++ b/tests/integration/test_accept_invalid_certificate/test.py @@ -1,9 +1,10 @@ -import pytest -from helpers.client import Client -from helpers.cluster import ClickHouseCluster import os.path from os import remove +import pytest + +from helpers.client import Client +from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MAX_RETRY = 5 diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 55cf3fa36b0..fd165461a94 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_access_for_functions/test.py b/tests/integration/test_access_for_functions/test.py index 138152904a8..ea76bf01813 100644 --- a/tests/integration/test_access_for_functions/test.py +++ b/tests/integration/test_access_for_functions/test.py @@ -1,5 +1,7 @@ -import pytest import uuid + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_aliases_in_default_expr_not_break_table_structure/test.py b/tests/integration/test_aliases_in_default_expr_not_break_table_structure/test.py index e0c15e18c23..04e9fb1c0b8 100644 --- a/tests/integration/test_aliases_in_default_expr_not_break_table_structure/test.py +++ b/tests/integration/test_aliases_in_default_expr_not_break_table_structure/test.py @@ -1,9 +1,9 @@ -import pytest import random import string -from helpers.cluster import ClickHouseCluster +import pytest +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_allowed_client_hosts/test.py b/tests/integration/test_allowed_client_hosts/test.py index 1fc9d0432cb..dfc0a434357 100644 --- a/tests/integration/test_allowed_client_hosts/test.py +++ b/tests/integration/test_allowed_client_hosts/test.py @@ -1,5 +1,7 @@ -import pytest import logging + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index df8934aa69b..fb6f4a209c5 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_alter_codec/test.py b/tests/integration/test_alter_codec/test.py index 7c7ef4803e9..01e07bbc704 100644 --- a/tests/integration/test_alter_codec/test.py +++ b/tests/integration/test_alter_codec/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_alter_comment_on_cluster/test.py b/tests/integration/test_alter_comment_on_cluster/test.py index 4cb10bbc751..5fbd98bfce7 100644 --- a/tests/integration/test_alter_comment_on_cluster/test.py +++ b/tests/integration/test_alter_comment_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_alter_moving_garbage/test.py b/tests/integration/test_alter_moving_garbage/test.py index 76b40b0df8d..c7cff571c17 100644 --- a/tests/integration/test_alter_moving_garbage/test.py +++ b/tests/integration/test_alter_moving_garbage/test.py @@ -1,9 +1,9 @@ import logging +import random +import threading import time import pytest -import threading -import random from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_alter_settings_on_cluster/test.py b/tests/integration/test_alter_settings_on_cluster/test.py index 7a552c383c9..3eca670d455 100644 --- a/tests/integration/test_alter_settings_on_cluster/test.py +++ b/tests/integration/test_alter_settings_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_alter_update_cast_keep_nullable/test.py b/tests/integration/test_alter_update_cast_keep_nullable/test.py index 71735888d69..76ef8307997 100644 --- a/tests/integration/test_alter_update_cast_keep_nullable/test.py +++ b/tests/integration/test_alter_update_cast_keep_nullable/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_alternative_keeper_config/test.py b/tests/integration/test_alternative_keeper_config/test.py index f1016ee1ae3..76b9e8a3f0c 100644 --- a/tests/integration/test_alternative_keeper_config/test.py +++ b/tests/integration/test_alternative_keeper_config/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_always_fetch_merged/test.py b/tests/integration/test_always_fetch_merged/test.py index 3bbfc8867f8..f8c44ce428a 100644 --- a/tests/integration/test_always_fetch_merged/test.py +++ b/tests/integration/test_always_fetch_merged/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py index 505d1629cd2..42bd140280b 100644 --- a/tests/integration/test_analyzer_compatibility/test.py +++ b/tests/integration/test_analyzer_compatibility/test.py @@ -1,7 +1,8 @@ -import uuid import time +import uuid import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_async_connect_to_multiple_ips/test.py b/tests/integration/test_async_connect_to_multiple_ips/test.py index acc4d24d0fa..bc57a941638 100644 --- a/tests/integration/test_async_connect_to_multiple_ips/test.py +++ b/tests/integration/test_async_connect_to_multiple_ips/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_async_insert_adaptive_busy_timeout/test.py b/tests/integration/test_async_insert_adaptive_busy_timeout/test.py index 280985b55b3..02e223709dc 100644 --- a/tests/integration/test_async_insert_adaptive_busy_timeout/test.py +++ b/tests/integration/test_async_insert_adaptive_busy_timeout/test.py @@ -1,12 +1,12 @@ import copy import logging -import pytest import random import timeit - +from itertools import repeat from math import floor from multiprocessing import Pool -from itertools import repeat + +import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_async_load_databases/test.py b/tests/integration/test_async_load_databases/test.py index 94aba46c713..7fc6fd222d1 100644 --- a/tests/integration/test_async_load_databases/test.py +++ b/tests/integration/test_async_load_databases/test.py @@ -1,5 +1,7 @@ -import pytest import random + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py b/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py index b3769a61b3f..248c9367b33 100644 --- a/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py +++ b/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_asynchronous_metric_log_table/test.py b/tests/integration/test_asynchronous_metric_log_table/test.py index efdd96004d2..52e42636547 100644 --- a/tests/integration/test_asynchronous_metric_log_table/test.py +++ b/tests/integration/test_asynchronous_metric_log_table/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_asynchronous_metrics_pk_bytes_fields/test.py b/tests/integration/test_asynchronous_metrics_pk_bytes_fields/test.py index 154048df35c..3c0befe8f00 100644 --- a/tests/integration/test_asynchronous_metrics_pk_bytes_fields/test.py +++ b/tests/integration/test_asynchronous_metrics_pk_bytes_fields/test.py @@ -1,5 +1,7 @@ -import pytest import time + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index 6ffa60de7b5..c8fc9b47d81 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_attach_backup_from_s3_plain/test.py b/tests/integration/test_attach_backup_from_s3_plain/test.py index c2f8936b82c..b6a79bdfbbb 100644 --- a/tests/integration/test_attach_backup_from_s3_plain/test.py +++ b/tests/integration/test_attach_backup_from_s3_plain/test.py @@ -2,6 +2,7 @@ # pylint: disable=line-too-long import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_attach_partition_using_copy/test.py b/tests/integration/test_attach_partition_using_copy/test.py index d5b07603dff..99fb2ff0153 100644 --- a/tests/integration/test_attach_partition_using_copy/test.py +++ b/tests/integration/test_attach_partition_using_copy/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_attach_table_from_s3_plain_readonly/test.py b/tests/integration/test_attach_table_from_s3_plain_readonly/test.py index 15ba934e621..96484a5e1eb 100644 --- a/tests/integration/test_attach_table_from_s3_plain_readonly/test.py +++ b/tests/integration/test_attach_table_from_s3_plain_readonly/test.py @@ -1,11 +1,12 @@ -import re -import os import logging +import os +import re +from pathlib import Path + import pytest +from minio.error import S3Error from helpers.cluster import ClickHouseCluster -from minio.error import S3Error -from pathlib import Path cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_attach_without_checksums/test.py b/tests/integration/test_attach_without_checksums/test.py index aee4b757efe..46f93c7b7e5 100644 --- a/tests/integration/test_attach_without_checksums/test.py +++ b/tests/integration/test_attach_without_checksums/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_attach_without_fetching/test.py b/tests/integration/test_attach_without_fetching/test.py index 67352e2dcbe..74b9a498b9c 100644 --- a/tests/integration/test_attach_without_fetching/test.py +++ b/tests/integration/test_attach_without_fetching/test.py @@ -1,11 +1,12 @@ -import time -import pytest import logging +import time + +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry -from helpers.network import PartitionManager from helpers.corrupt_part_data_on_disk import corrupt_part_data_by_path +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry def fill_node(node): diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index 38be07eca49..7f0f0318c0c 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_azure_blob_storage_native_copy/test.py b/tests/integration/test_azure_blob_storage_native_copy/test.py index 77d400240b1..2819da15936 100644 --- a/tests/integration/test_azure_blob_storage_native_copy/test.py +++ b/tests/integration/test_azure_blob_storage_native_copy/test.py @@ -1,20 +1,21 @@ #!/usr/bin/env python3 import gzip +import io import json import logging import os -import io import random import threading import time -from azure.storage.blob import BlobServiceClient -import helpers.client import pytest +from azure.storage.blob import BlobServiceClient + +import helpers.client from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.network import PartitionManager from helpers.mock_servers import start_mock_servers +from helpers.network import PartitionManager from helpers.test_tools import exec_query_with_retry diff --git a/tests/integration/test_azure_blob_storage_plain_rewritable/test.py b/tests/integration/test_azure_blob_storage_plain_rewritable/test.py index 96d116ec6a2..157232eadba 100644 --- a/tests/integration/test_azure_blob_storage_plain_rewritable/test.py +++ b/tests/integration/test_azure_blob_storage_plain_rewritable/test.py @@ -4,9 +4,9 @@ import random import string import pytest +from azure.storage.blob import BlobServiceClient from helpers.cluster import ClickHouseCluster -from azure.storage.blob import BlobServiceClient from test_storage_azure_blob_storage.test import azure_query NODE_NAME = "node" diff --git a/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py b/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py index d3555c19b39..1f5650a51e6 100644 --- a/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py +++ b/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py @@ -1,9 +1,10 @@ import logging -import pytest -from helpers.cluster import ClickHouseCluster -from test_storage_azure_blob_storage.test import azure_query import os +import pytest + +from helpers.cluster import ClickHouseCluster +from test_storage_azure_blob_storage.test import azure_query logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) diff --git a/tests/integration/test_backup_log/test.py b/tests/integration/test_backup_log/test.py index 2bab94226e1..74d3af5ac53 100644 --- a/tests/integration/test_backup_log/test.py +++ b/tests/integration/test_backup_log/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index b91a7cfe1c8..149dff943b5 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -1,7 +1,8 @@ +import logging import os.path import pytest -import logging + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 78b186e3227..407fc2d4a7b 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -1,20 +1,21 @@ #!/usr/bin/env python3 import gzip +import io import json import logging import os -import io import random import threading import time -from azure.storage.blob import BlobServiceClient -import helpers.client import pytest +from azure.storage.blob import BlobServiceClient + +import helpers.client from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.network import PartitionManager from helpers.mock_servers import start_mock_servers +from helpers.network import PartitionManager from helpers.test_tools import exec_query_with_retry diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py index c401f482c3f..67b5406c7c1 100644 --- a/tests/integration/test_backup_restore_keeper_map/test.py +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -1,7 +1,8 @@ from time import sleep -import pytest -from helpers.cluster import ClickHouseCluster +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 211a03324c4..eca425b5dcc 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1,14 +1,15 @@ import glob import os.path -import pytest import random import re import sys import uuid from collections import namedtuple -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry script_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_backup_restore_new/test_cancel_backup.py b/tests/integration/test_backup_restore_new/test_cancel_backup.py index cce23a7e932..dc7ef0b0086 100644 --- a/tests/integration/test_backup_restore_new/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_new/test_cancel_backup.py @@ -1,9 +1,10 @@ +import re +import uuid + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, assert_eq_with_retry -import uuid -import re - cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backup_restore_new/test_shutdown_wait_backup.py b/tests/integration/test_backup_restore_new/test_shutdown_wait_backup.py index 326e0c5da6c..cd335135175 100644 --- a/tests/integration/test_backup_restore_new/test_shutdown_wait_backup.py +++ b/tests/integration/test_backup_restore_new/test_shutdown_wait_backup.py @@ -1,8 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV, assert_eq_with_retry import uuid +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 6d578f37bf5..a1082c563d1 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -1,11 +1,13 @@ -import pytest -import re import os.path -import random, string +import random +import re +import string + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, assert_eq_with_retry - cluster = ClickHouseCluster(__file__) main_configs = [ diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py index 0e381d48fb1..0caa9c3272d 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py @@ -1,12 +1,13 @@ -from random import random, randint -import pytest +import concurrent import os.path import time -import concurrent +from random import randint, random + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, assert_eq_with_retry - cluster = ClickHouseCluster(__file__) num_nodes = 10 diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index f3b6993c4fd..846c41592f7 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -1,11 +1,13 @@ -from random import randint -import pytest -import os.path -import time import concurrent +import os.path +import re +import time +from random import randint + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, assert_eq_with_retry -import re cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py index 15c344eadf8..9f8a42dfd01 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py +++ b/tests/integration/test_backup_restore_on_cluster/test_slow_rmt.py @@ -3,7 +3,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, assert_eq_with_retry, exec_query_with_retry - cluster = ClickHouseCluster(__file__) main_configs = [ diff --git a/tests/integration/test_backup_restore_on_cluster/test_two_shards_two_replicas.py b/tests/integration/test_backup_restore_on_cluster/test_two_shards_two_replicas.py index c0e318c8bb7..8cccbf3a7c3 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_two_shards_two_replicas.py +++ b/tests/integration/test_backup_restore_on_cluster/test_two_shards_two_replicas.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) main_configs = [ diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 381268ce7fe..f58190522a3 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1,10 +1,11 @@ +import os +import uuid from typing import Dict + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -import uuid -import os - CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_backup_restore_storage_policy/test.py b/tests/integration/test_backup_restore_storage_policy/test.py index 4e9d309a220..9f6795a8cdd 100644 --- a/tests/integration/test_backup_restore_storage_policy/test.py +++ b/tests/integration/test_backup_restore_storage_policy/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster backup_id_counter = 0 diff --git a/tests/integration/test_backup_s3_storage_class/test.py b/tests/integration/test_backup_s3_storage_class/test.py index 2b11f20afc6..a0aeda67f38 100644 --- a/tests/integration/test_backup_s3_storage_class/test.py +++ b/tests/integration/test_backup_s3_storage_class/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 7de5f51921b..0f9a57c7bbc 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index 6b385bf8402..e4433fac30f 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py index 9878c1ed70e..bb18244308b 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index f5d0c066600..0a29f443362 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -1,5 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index e0be009e874..3fb0d3cc428 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index acf4bd28c9b..74f3f9b128d 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -4,9 +4,11 @@ # pylint: disable=redefined-outer-name import logging + import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION + from helpers.client import QueryRuntimeException +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) upstream = cluster.add_instance("upstream", use_old_analyzer=True) diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index a90453d045b..dd1459a2937 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -4,7 +4,7 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) upstream_node = cluster.add_instance("upstream_node", use_old_analyzer=True) diff --git a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py index 4752a589a44..8631bcc3f09 100644 --- a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py +++ b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) # Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index b13e6c975e8..7b9f87d269d 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 83be0e4c5a3..5255f3c4265 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) diff --git a/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py index e1b9049ef5d..b9e68a5fce7 100644 --- a/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py +++ b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py @@ -2,7 +2,6 @@ import pytest from helpers.cluster import ClickHouseCluster - cluster = ClickHouseCluster(__file__) cluster_name = "parallel_replicas" nodes = [ diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index cbe147dc07b..b900d97a793 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index ad573c7ffe0..3a1ffaf7b48 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION, is_arm +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster, is_arm # For arm version see https://github.com/ClickHouse/ClickHouse/pull/59132 cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index e36c3310e4a..599f5bf6557 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_broken_part_during_merge/test.py b/tests/integration/test_broken_part_during_merge/test.py index 0ba7beeb1fd..5abd70ab95e 100644 --- a/tests/integration/test_broken_part_during_merge/test.py +++ b/tests/integration/test_broken_part_during_merge/test.py @@ -1,9 +1,10 @@ +import time +from multiprocessing.dummy import Pool + import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk -import time cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index 76cbdbf9e57..3b0a4d57c3f 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -1,11 +1,13 @@ -import time -import pytest import logging -import string import random -from helpers.cluster import ClickHouseCluster +import string +import time from multiprocessing.dummy import Pool +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_buffer_profile/test.py b/tests/integration/test_buffer_profile/test.py index b1185493c47..0efc70e1306 100644 --- a/tests/integration/test_buffer_profile/test.py +++ b/tests/integration/test_buffer_profile/test.py @@ -4,8 +4,8 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_build_sets_from_multiple_threads/test.py b/tests/integration/test_build_sets_from_multiple_threads/test.py index df91bcd1c4b..200bdf567ca 100644 --- a/tests/integration/test_build_sets_from_multiple_threads/test.py +++ b/tests/integration/test_build_sets_from_multiple_threads/test.py @@ -2,10 +2,11 @@ # pylint: disable=redefined-outer-name # pylint: disable=line-too-long -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", user_configs=["configs/users_overrides.xml"]) diff --git a/tests/integration/test_cgroup_limit/test.py b/tests/integration/test_cgroup_limit/test.py index e77b0f70960..f75bbd41ef7 100644 --- a/tests/integration/test_cgroup_limit/test.py +++ b/tests/integration/test_cgroup_limit/test.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 import logging -import os import math +import os import subprocess from tempfile import NamedTemporaryFile + import pytest diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py index 021977fb6b6..774d322d7e0 100644 --- a/tests/integration/test_check_table/test.py +++ b/tests/integration/test_check_table/test.py @@ -1,8 +1,9 @@ +import concurrent + import pytest -import concurrent -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 76a0f30f82e..92456f6c6d5 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import logging -import pytest import os -import minio import random import string +import minio +import pytest + from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_s3_mock from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_cleanup_after_start/test.py b/tests/integration/test_cleanup_after_start/test.py index f2d7af440ab..fcdd789293c 100644 --- a/tests/integration/test_cleanup_after_start/test.py +++ b/tests/integration/test_cleanup_after_start/test.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 +import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_logs_contain_with_retry -import os cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True, stay_alive=True) diff --git a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py index 3c59d99b7fc..ad39aefe1c1 100644 --- a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py +++ b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index 9797db7c498..2ee1504b073 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -1,7 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_cluster_discovery/test.py b/tests/integration/test_cluster_discovery/test.py index ab21c72cec4..b47c8d06269 100644 --- a/tests/integration/test_cluster_discovery/test.py +++ b/tests/integration/test_cluster_discovery/test.py @@ -1,11 +1,11 @@ +import functools + import pytest -import functools -from .common import check_on_cluster - - from helpers.cluster import ClickHouseCluster +from .common import check_on_cluster + cluster = ClickHouseCluster(__file__) shard_configs = { diff --git a/tests/integration/test_cluster_discovery/test_password.py b/tests/integration/test_cluster_discovery/test_password.py index bf593260d6f..7f65fce363f 100644 --- a/tests/integration/test_cluster_discovery/test_password.py +++ b/tests/integration/test_cluster_discovery/test_password.py @@ -1,9 +1,9 @@ import pytest -from .common import check_on_cluster - from helpers.cluster import ClickHouseCluster +from .common import check_on_cluster + cluster = ClickHouseCluster(__file__) nodes = { diff --git a/tests/integration/test_codec_encrypted/test.py b/tests/integration/test_codec_encrypted/test.py index 64b8bf29640..b23ab8821c2 100644 --- a/tests/integration/test_codec_encrypted/test.py +++ b/tests/integration/test_codec_encrypted/test.py @@ -1,6 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_composable_protocols/test.py b/tests/integration/test_composable_protocols/test.py index 241d1505433..5748f22b845 100644 --- a/tests/integration/test_composable_protocols/test.py +++ b/tests/integration/test_composable_protocols/test.py @@ -1,14 +1,17 @@ -import ssl -import pytest -import os.path as p import os -from helpers.cluster import ClickHouseCluster -from helpers.client import Client -import urllib.request, urllib.parse -import subprocess +import os.path as p import socket +import ssl +import subprocess +import urllib.parse +import urllib.request import warnings +import pytest + +from helpers.client import Client +from helpers.cluster import ClickHouseCluster + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_compressed_marks_restart/test.py b/tests/integration/test_compressed_marks_restart/test.py index 90e09d62792..c656c706d7a 100644 --- a/tests/integration/test_compressed_marks_restart/test.py +++ b/tests/integration/test_compressed_marks_restart/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_compression_nested_columns/test.py b/tests/integration/test_compression_nested_columns/test.py index 3541a9f6061..0b9a874ccfc 100644 --- a/tests/integration/test_compression_nested_columns/test.py +++ b/tests/integration/test_compression_nested_columns/test.py @@ -2,6 +2,7 @@ import random import string import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_concurrent_backups_s3/test.py b/tests/integration/test_concurrent_backups_s3/test.py index 312ebdba5e3..66986e9cbe8 100644 --- a/tests/integration/test_concurrent_backups_s3/test.py +++ b/tests/integration/test_concurrent_backups_s3/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest -import re import os.path +import re +import time from multiprocessing.dummy import Pool + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -import time cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py b/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py index 166724a7f8c..90cb001db3e 100644 --- a/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py +++ b/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py @@ -2,6 +2,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_concurrent_queries_for_user_restriction/test.py b/tests/integration/test_concurrent_queries_for_user_restriction/test.py index c4afdf99685..0395e82de80 100644 --- a/tests/integration/test_concurrent_queries_for_user_restriction/test.py +++ b/tests/integration/test_concurrent_queries_for_user_restriction/test.py @@ -2,6 +2,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py index 830090a1c0d..7e2fec9230e 100644 --- a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py +++ b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py @@ -2,8 +2,8 @@ import time from multiprocessing.dummy import Pool import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node_insert = cluster.add_instance( diff --git a/tests/integration/test_concurrent_threads_soft_limit/test.py b/tests/integration/test_concurrent_threads_soft_limit/test.py index d1e233ee12f..02655be61f3 100644 --- a/tests/integration/test_concurrent_threads_soft_limit/test.py +++ b/tests/integration/test_concurrent_threads_soft_limit/test.py @@ -1,9 +1,10 @@ -import pytest -from helpers.cluster import ClickHouseCluster - import threading import time + +import pytest + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index 3a3981d65ba..9534fb78af3 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -1,9 +1,10 @@ -import time import logging +import time import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_config_corresponding_root/test.py b/tests/integration/test_config_corresponding_root/test.py index f4ec1f1e658..b6d4326b66e 100644 --- a/tests/integration/test_config_corresponding_root/test.py +++ b/tests/integration/test_config_corresponding_root/test.py @@ -1,6 +1,7 @@ import os import pytest + from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_config_decryption/test.py b/tests/integration/test_config_decryption/test.py index dd8cdc2e4e1..5ee430b363f 100644 --- a/tests/integration/test_config_decryption/test.py +++ b/tests/integration/test_config_decryption/test.py @@ -1,5 +1,7 @@ -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_config_decryption/test_wrong_settings.py b/tests/integration/test_config_decryption/test_wrong_settings.py index c6987d12324..f66390709e5 100644 --- a/tests/integration/test_config_decryption/test_wrong_settings.py +++ b/tests/integration/test_config_decryption/test_wrong_settings.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_config_hide_in_preprocessed/test.py b/tests/integration/test_config_hide_in_preprocessed/test.py index fd237063b18..a7bec10dd19 100644 --- a/tests/integration/test_config_hide_in_preprocessed/test.py +++ b/tests/integration/test_config_hide_in_preprocessed/test.py @@ -1,7 +1,8 @@ -import pytest import os -from helpers.cluster import ClickHouseCluster +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_config_reloader_interval/test.py b/tests/integration/test_config_reloader_interval/test.py index 22b66ecac30..0743925155d 100644 --- a/tests/integration/test_config_reloader_interval/test.py +++ b/tests/integration/test_config_reloader_interval/test.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -import pytest import fnmatch -from helpers.cluster import ClickHouseCluster +import pytest + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 124dbcaedf7..bb208bcdd86 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -1,7 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_config_yaml_merge_keys/test.py b/tests/integration/test_config_yaml_merge_keys/test.py index e7ee164f170..b48bf5058e7 100644 --- a/tests/integration/test_config_yaml_merge_keys/test.py +++ b/tests/integration/test_config_yaml_merge_keys/test.py @@ -1,5 +1,6 @@ -import helpers import pytest + +import helpers from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_covered_by_broken_exists/test.py b/tests/integration/test_covered_by_broken_exists/test.py index caa091fdd2d..d0278186141 100644 --- a/tests/integration/test_covered_by_broken_exists/test.py +++ b/tests/integration/test_covered_by_broken_exists/test.py @@ -1,9 +1,10 @@ -import pytest import logging import time + +import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", stay_alive=True, with_zookeeper=True) diff --git a/tests/integration/test_crash_log/test.py b/tests/integration/test_crash_log/test.py index 5a63e6ca6a7..ce126485670 100644 --- a/tests/integration/test_crash_log/test.py +++ b/tests/integration/test_crash_log/test.py @@ -1,5 +1,6 @@ import os import time + import pytest import helpers.cluster diff --git a/tests/integration/test_create_query_constraints/test.py b/tests/integration/test_create_query_constraints/test.py index 33c41b4f161..57efd66c343 100644 --- a/tests/integration/test_create_query_constraints/test.py +++ b/tests/integration/test_create_query_constraints/test.py @@ -1,10 +1,12 @@ -import pytest import asyncio -import re -import random import os.path +import random +import re + +import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py index b60ec65cb7b..979c061f845 100644 --- a/tests/integration/test_create_user_and_login/test.py +++ b/tests/integration/test_create_user_and_login/test.py @@ -1,6 +1,8 @@ -import pytest -import time import logging +import time + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_cross_replication/test.py b/tests/integration/test_cross_replication/test.py index 2a73acadafd..c2e896f83d9 100644 --- a/tests/integration/test_cross_replication/test.py +++ b/tests/integration/test_cross_replication/test.py @@ -2,6 +2,7 @@ import time from contextlib import contextmanager import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_custom_dashboards/test.py b/tests/integration/test_custom_dashboards/test.py index 7b360f58dc7..21f2fa8b66e 100644 --- a/tests/integration/test_custom_dashboards/test.py +++ b/tests/integration/test_custom_dashboards/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_custom_settings/test.py b/tests/integration/test_custom_settings/test.py index 2dd4a7dafef..07a0a3f6127 100644 --- a/tests/integration/test_custom_settings/test.py +++ b/tests/integration/test_custom_settings/test.py @@ -1,5 +1,7 @@ -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_ddl_worker_non_leader/test.py b/tests/integration/test_ddl_worker_non_leader/test.py index 3fdd180c635..daaf13a6feb 100644 --- a/tests/integration/test_ddl_worker_non_leader/test.py +++ b/tests/integration/test_ddl_worker_non_leader/test.py @@ -1,8 +1,10 @@ -import pytest import time + +import pytest + +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 4fe899f15e2..8c2e13599c7 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -1,8 +1,10 @@ +import logging import random import string -import logging -import pytest import time + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_default_database_on_cluster/test.py b/tests/integration/test_default_database_on_cluster/test.py index f0f1360434f..de19f75d8d7 100644 --- a/tests/integration/test_default_database_on_cluster/test.py +++ b/tests/integration/test_default_database_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_default_role/test.py b/tests/integration/test_default_role/test.py index 1a321a8269a..1f023e61def 100644 --- a/tests/integration/test_default_role/test.py +++ b/tests/integration/test_default_role/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_detached_parts_metrics/test.py b/tests/integration/test_detached_parts_metrics/test.py index fb312f8d224..0fe695b1e11 100644 --- a/tests/integration/test_detached_parts_metrics/test.py +++ b/tests/integration/test_detached_parts_metrics/test.py @@ -1,10 +1,13 @@ import time + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts - +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, +) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py index 993c8259f32..0af64b10187 100644 --- a/tests/integration/test_dictionaries_access/test.py +++ b/tests/integration/test_dictionaries_access/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py index 01addae2542..4df0cde27f5 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py @@ -1,7 +1,7 @@ import os import shutil -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row KEY_FIELDS = { "simple": [Field("KeyField", "UInt64", is_key=True, default_value_for_get=9999999)], diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py index bb0e3b47414..42e1adba1c4 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceClickHouse +from .common import * + SOURCE = SourceClickHouse( "LocalClickHouse", "localhost", "9000", "local_node", "9000", "default", "" ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py index bf4d05a154c..f3721e88790 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceClickHouse +from .common import * + SOURCE = SourceClickHouse( "RemoteClickHouse", "localhost", "9000", "clickhouse_remote", "9000", "default", "" ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py index 6af5fa841c1..71e1e3555c8 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceExecutableCache +from .common import * + SOURCE = SourceExecutableCache( "ExecutableCache", "localhost", "9000", "cache_node", "9000", "", "" ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py index dfcf1e4fc64..e969826f6be 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceExecutableHashed +from .common import * + SOURCE = SourceExecutableHashed( "ExecutableHashed", "localhost", "9000", "hashed_node", "9000", "", "" ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py index e9bf93b3c8e..22ef33f8d6e 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceFile +from .common import * + SOURCE = SourceFile("File", "localhost", "9000", "file_node", "9000", "", "") cluster = None diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py index 94220d7c698..0d63ab6fb83 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceHTTP +from .common import * + SOURCE = SourceHTTP("SourceHTTP", "localhost", "9000", "clickhouse_h", "9000", "", "") cluster = None diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py index 0b7476faf2e..c2f1f523344 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceHTTPS +from .common import * + SOURCE = SourceHTTPS( "SourceHTTPS", "localhost", "9000", "clickhouse_hs", "9000", "", "" ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py index 163c5479596..d188705cf6c 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -1,11 +1,12 @@ import os -import pytest -from .common import * +import pytest from helpers.cluster import ClickHouseCluster from helpers.external_sources import SourceMongo +from .common import * + simple_tester = None complex_tester = None ranged_tester = None diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index 700d4b550c1..bdaebab1149 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceMongoURI +from .common import * + test_name = "mongo_uri" diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py index afae8d616b0..792fe8cf279 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py @@ -1,13 +1,14 @@ -import os import math +import os + import pytest -from .common import * - from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceMySQL +from .common import * + SOURCE = None cluster = None node = None diff --git a/tests/integration/test_dictionaries_complex_key_cache_string/test.py b/tests/integration/test_dictionaries_complex_key_cache_string/test.py index bca7122a2a8..193b8dc037b 100644 --- a/tests/integration/test_dictionaries_complex_key_cache_string/test.py +++ b/tests/integration/test_dictionaries_complex_key_cache_string/test.py @@ -1,6 +1,7 @@ import os import pytest + from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_dictionaries_config_reload/test.py b/tests/integration/test_dictionaries_config_reload/test.py index 7be179f854b..6d0f6edc01f 100644 --- a/tests/integration/test_dictionaries_config_reload/test.py +++ b/tests/integration/test_dictionaries_config_reload/test.py @@ -1,7 +1,8 @@ +import logging import os import sys import time -import logging + import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 2f31e406735..bbce196400d 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -1,10 +1,11 @@ +import logging import os -import warnings import time +import warnings import pymysql import pytest -import logging + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index 4ce55105436..7bf763e6443 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionaries_dependency_xml/test.py b/tests/integration/test_dictionaries_dependency_xml/test.py index 3f4c3320920..cf8e0baad58 100644 --- a/tests/integration/test_dictionaries_dependency_xml/test.py +++ b/tests/integration/test_dictionaries_dependency_xml/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index 332f4ca11bb..0dd62d819ec 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -1,10 +1,12 @@ ## sudo -H pip install PyMySQL +import logging +import time import warnings + import pymysql.cursors import pytest + from helpers.cluster import ClickHouseCluster -import time -import logging DICTS = ["configs/dictionaries/mysql_dict1.xml", "configs/dictionaries/mysql_dict2.xml"] CONFIG_FILES = ["configs/remote_servers.xml", "configs/named_collections.xml"] diff --git a/tests/integration/test_dictionaries_null_value/test.py b/tests/integration/test_dictionaries_null_value/test.py index d62b1e6fc49..96dbc7487ab 100644 --- a/tests/integration/test_dictionaries_null_value/test.py +++ b/tests/integration/test_dictionaries_null_value/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster DICTIONARY_FILES = ["configs/dictionaries/cache.xml"] diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 010ecdb5084..f8f07d04c62 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -1,12 +1,13 @@ -import pytest -import time import logging -import psycopg2 +import time from multiprocessing.dummy import Pool +import psycopg2 +import pytest +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + from helpers.cluster import ClickHouseCluster from helpers.postgres_utility import get_postgres_conn -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_dictionaries_redis/test.py b/tests/integration/test_dictionaries_redis/test.py index c2dc73db782..b54fb9967a7 100644 --- a/tests/integration/test_dictionaries_redis/test.py +++ b/tests/integration/test_dictionaries_redis/test.py @@ -1,9 +1,11 @@ +import logging import os import shutil + import pytest -import logging + from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.dictionary import Dictionary, DictionaryStructure, Field, Layout, Row from helpers.external_sources import SourceRedis cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionaries_redis/test_long.py b/tests/integration/test_dictionaries_redis/test_long.py index beb9513884a..54066e017f0 100644 --- a/tests/integration/test_dictionaries_redis/test_long.py +++ b/tests/integration/test_dictionaries_redis/test_long.py @@ -1,7 +1,8 @@ import pytest -from helpers.cluster import ClickHouseCluster import redis +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", with_redis=True) diff --git a/tests/integration/test_dictionaries_replace/test.py b/tests/integration/test_dictionaries_replace/test.py index efb65c15c49..517cc2afe02 100644 --- a/tests/integration/test_dictionaries_replace/test.py +++ b/tests/integration/test_dictionaries_replace/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) node_ll = cluster.add_instance( diff --git a/tests/integration/test_dictionaries_select_all/test.py b/tests/integration/test_dictionaries_select_all/test.py index 0a740394129..1f0f499e7d4 100644 --- a/tests/integration/test_dictionaries_select_all/test.py +++ b/tests/integration/test_dictionaries_select_all/test.py @@ -1,13 +1,14 @@ import os import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV from .generate_dictionaries import ( - generate_structure, - generate_dictionaries, DictionaryTestTable, + generate_dictionaries, + generate_structure, ) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index 3ed854d2c9f..7594f756b1f 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -2,6 +2,7 @@ import os import time import pytest + from helpers.client import QueryTimeoutExceedException from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_dictionaries_wait_for_load/test.py b/tests/integration/test_dictionaries_wait_for_load/test.py index b30cc61abce..9e0fe2346c0 100644 --- a/tests/integration/test_dictionaries_wait_for_load/test.py +++ b/tests/integration/test_dictionaries_wait_for_load/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - DICTIONARY_FILES = [ "dictionaries/long_loading_dictionary.xml", ] diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py index 921fb4e4154..984fb194c7d 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py @@ -1,8 +1,8 @@ import time import pytest -from helpers.cluster import ClickHouseCluster -from helpers.cluster import ClickHouseKiller + +from helpers.cluster import ClickHouseCluster, ClickHouseKiller from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py index 6eedf63f95c..fe850afcead 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py @@ -4,6 +4,7 @@ import string import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py index 8e45af44640..663d8e64ce9 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py @@ -1,8 +1,8 @@ import time import pytest -from helpers.cluster import ClickHouseCluster -from helpers.cluster import ClickHouseKiller + +from helpers.cluster import ClickHouseCluster, ClickHouseKiller from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py index 8d6242f4711..78c0d1bd3cf 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py @@ -1,8 +1,8 @@ import time import pytest -from helpers.cluster import ClickHouseCluster -from helpers.cluster import ClickHouseKiller + +from helpers.cluster import ClickHouseCluster, ClickHouseKiller from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionary_custom_settings/test.py b/tests/integration/test_dictionary_custom_settings/test.py index eb394da8bb6..46123f513f6 100644 --- a/tests/integration/test_dictionary_custom_settings/test.py +++ b/tests/integration/test_dictionary_custom_settings/test.py @@ -1,10 +1,10 @@ import os import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - DICTIONARY_FILES = [ "configs/dictionaries/FileSourceConfig.xml", "configs/dictionaries/ExecutableSourceConfig.xml", diff --git a/tests/integration/test_dictionary_ddl_on_cluster/test.py b/tests/integration/test_dictionary_ddl_on_cluster/test.py index dc9d31d75bd..e22b2a986a3 100644 --- a/tests/integration/test_dictionary_ddl_on_cluster/test.py +++ b/tests/integration/test_dictionary_ddl_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py index b6da7ed548f..8b196b97a2c 100644 --- a/tests/integration/test_disable_insertion_and_mutation/test.py +++ b/tests/integration/test_disable_insertion_and_mutation/test.py @@ -1,7 +1,9 @@ +import time + import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -import time cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_disabled_access_control_improvements/test_row_policy.py b/tests/integration/test_disabled_access_control_improvements/test_row_policy.py index c09a80cea06..25039f44532 100644 --- a/tests/integration/test_disabled_access_control_improvements/test_row_policy.py +++ b/tests/integration/test_disabled_access_control_improvements/test_row_policy.py @@ -1,5 +1,7 @@ import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py index 4ea5a524fed..c8c3cd32e5d 100644 --- a/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py +++ b/tests/integration/test_disabled_access_control_improvements/test_select_from_system_tables.py @@ -1,5 +1,7 @@ import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py index 814aebb0d8e..ea375b197fa 100644 --- a/tests/integration/test_disabled_mysql_server/test.py +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -1,9 +1,10 @@ -import time import contextlib -import pymysql.cursors -import pytest import os import subprocess +import time + +import pymysql.cursors +import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, get_docker_compose_path diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index ae756ef1e27..132a0e41ae3 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index f297c665dc5..0f4ea182d64 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index ec0bef23731..1aa1ff08e0d 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster uuids = [] diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index a5e2456ef4f..db80c93e8a0 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index a4b2399e117..06793c09d86 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -1,7 +1,7 @@ -from helpers.cluster import ClickHouseCluster - import pytest +from helpers.cluster import ClickHouseCluster + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_disks_app_interactive/test.py b/tests/integration/test_disks_app_interactive/test.py index ca4ba5d9065..3163520b220 100644 --- a/tests/integration/test_disks_app_interactive/test.py +++ b/tests/integration/test_disks_app_interactive/test.py @@ -1,15 +1,13 @@ -from helpers.cluster import ClickHouseCluster +import io +import os +import pathlib +import select +import subprocess +from typing import Dict, List, Optional, Tuple, Union import pytest -import pathlib - -import subprocess -import select -import io -from typing import List, Tuple, Dict, Union, Optional - -import os +from helpers.cluster import ClickHouseCluster class ClickHouseDisksException(Exception): diff --git a/tests/integration/test_distributed_async_insert_for_node_changes/test.py b/tests/integration/test_distributed_async_insert_for_node_changes/test.py index 6bd6341dcc8..5b9370b3d53 100644 --- a/tests/integration/test_distributed_async_insert_for_node_changes/test.py +++ b/tests/integration/test_distributed_async_insert_for_node_changes/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_config/test.py b/tests/integration/test_distributed_config/test.py index e551e69b93f..5c2d3978f6b 100644 --- a/tests/integration/test_distributed_config/test.py +++ b/tests/integration/test_distributed_config/test.py @@ -1,7 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster import logging +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", main_configs=["configs/overrides.xml", "configs/clusters.xml"] diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index f08c6265b82..e65a678a8a5 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -8,6 +8,7 @@ import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from helpers.test_tools import TSV + from .cluster import ClickHouseClusterWithDDLHelpers @@ -21,7 +22,9 @@ def test_cluster(request): yield cluster instance = cluster.instances["ch1"] - cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") + cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test ON CLUSTER 'cluster'" + ) cluster.ddl_check_query( instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'" ) @@ -48,7 +51,7 @@ def test_default_database(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null", + "CREATE TABLE IF NOT EXISTS null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null", settings={"distributed_ddl_entry_format_version": 2}, ) @@ -74,14 +77,15 @@ def test_create_view(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV", + "CREATE VIEW IF NOT EXISTS test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV", ) test_cluster.ddl_check_query( instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV", ) test_cluster.ddl_check_query( - instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV" + instance, + "DROP TABLE IF EXISTS test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV", ) test_cluster.ddl_check_query( instance, @@ -90,14 +94,14 @@ def test_create_view(test_cluster): test_cluster.ddl_check_query( instance, - "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory", + "CREATE TABLE IF NOT EXISTS test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory", ) test_cluster.ddl_check_query( instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV", ) test_cluster.ddl_check_query( - instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'" + instance, "DROP TABLE IF EXISTS test.super_simple2 ON CLUSTER 'cluster'" ) @@ -130,7 +134,7 @@ def test_on_server_fail(test_cluster): assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") test_cluster.ddl_check_query( - instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'" + instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'" ) @@ -210,12 +214,14 @@ ENGINE = Distributed('{cluster}', default, merge, i) "".join(["{}\t{}\n".format(x, x) for x in range(4)]) ) - test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'") test_cluster.ddl_check_query( - instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'" + instance, "DROP TABLE IF EXISTS merge ON CLUSTER '{cluster}'" ) test_cluster.ddl_check_query( - instance, "DROP TABLE all_merge_64 ON CLUSTER '{cluster}'" + instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER '{cluster}'" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER '{cluster}'" ) @@ -223,7 +229,7 @@ def test_macro(test_cluster): instance = test_cluster.instances["ch2"] test_cluster.ddl_check_query( instance, - "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory", + "CREATE TABLE IF NOT EXISTS tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory", ) for i in range(4): @@ -234,7 +240,7 @@ def test_macro(test_cluster): test_cluster.ddl_check_query( instance, - "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)", + "CREATE TABLE IF NOT EXISTS distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)", ) assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV( @@ -350,7 +356,7 @@ def test_optimize_query(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)", + "CREATE TABLE IF NOT EXISTS test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)", ) test_cluster.ddl_check_query( instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV" @@ -361,7 +367,7 @@ def test_create_as_select(test_cluster): instance = test_cluster.instances["ch2"] test_cluster.ddl_check_query( instance, - "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)", + "CREATE TABLE IF NOT EXISTS test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)", ) assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV( "1\n2\n" @@ -375,11 +381,11 @@ def test_create_reserved(test_cluster): instance = test_cluster.instances["ch2"] test_cluster.ddl_check_query( instance, - "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)", + "CREATE TABLE IF NOT EXISTS test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)", ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)", + "CREATE TABLE IF NOT EXISTS test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)", ) test_cluster.ddl_check_query( instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster" @@ -407,11 +413,11 @@ def test_rename(test_cluster): test_cluster.ddl_check_query( instance, - "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)", + "CREATE TABLE IF NOT EXISTS rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)", ) test_cluster.ddl_check_query( instance, - "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", + "CREATE TABLE IF NOT EXISTS rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", ) test_cluster.ddl_check_query( instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;" @@ -433,7 +439,7 @@ def test_rename(test_cluster): test_cluster.ddl_check_query( instance, - "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", + "CREATE TABLE IF NOT EXISTS rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", ) instance.query("system stop distributed sends rename") @@ -493,7 +499,8 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( - instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic" + instance, + "CREATE DATABASE IF NOT EXISTS test_atomic ON CLUSTER cluster ENGINE=Atomic", ) assert ( "are supported only for ON CLUSTER queries with Atomic database engine" @@ -503,14 +510,14 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree() ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree() ORDER BY n", ) test_cluster.ddl_check_query( instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster SYNC" ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rmt UUID '12345678-0000-4000-8000-000000000001' ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rmt UUID '12345678-0000-4000-8000-000000000001' ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", ) assert ( instance.query("SHOW CREATE test_atomic.rmt FORMAT TSVRaw") @@ -522,7 +529,7 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY n", ) test_cluster.ddl_check_query( instance, @@ -542,15 +549,15 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rrmt ON CLUSTER cluster (n UInt64, m UInt64) ENGINE=ReplicatedReplacingMergeTree(m) ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rrmt ON CLUSTER cluster (n UInt64, m UInt64) ENGINE=ReplicatedReplacingMergeTree(m) ORDER BY n", ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rsmt ON CLUSTER cluster (n UInt64, m UInt64, k UInt64) ENGINE=ReplicatedSummingMergeTree((m, k)) ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rsmt ON CLUSTER cluster (n UInt64, m UInt64, k UInt64) ENGINE=ReplicatedSummingMergeTree((m, k)) ORDER BY n", ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n", ) test_cluster.ddl_check_query( instance, "DROP DATABASE test_atomic ON CLUSTER cluster SYNC" @@ -558,7 +565,7 @@ def test_replicated_without_arguments(test_cluster): test_cluster.ddl_check_query( instance, - "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary", + "CREATE DATABASE IF NOT EXISTS test_ordinary ON CLUSTER cluster ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1}, ) assert ( @@ -575,14 +582,14 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( instance, - "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n", + "CREATE TABLE IF NOT EXISTS test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n", ) assert ( instance.query("SHOW CREATE test_ordinary.rmt FORMAT TSVRaw") == "CREATE TABLE test_ordinary.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/{shard}/rmt/', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" ) test_cluster.ddl_check_query( - instance, "DROP DATABASE test_ordinary ON CLUSTER cluster SYNC" + instance, "DROP DATABASE IF EXISTS test_ordinary ON CLUSTER cluster SYNC" ) test_cluster.pm_random_drops.push_rules(rules) diff --git a/tests/integration/test_distributed_ddl/test_replicated_alter.py b/tests/integration/test_distributed_ddl/test_replicated_alter.py index 08d2c1da278..f77248fd485 100644 --- a/tests/integration/test_distributed_ddl/test_replicated_alter.py +++ b/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -7,6 +7,7 @@ import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from helpers.test_tools import TSV + from .cluster import ClickHouseClusterWithDDLHelpers diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/test.py b/tests/integration/test_distributed_ddl_on_cross_replication/test.py index a85975e1ba6..3efee55c9b7 100644 --- a/tests/integration/test_distributed_ddl_on_cross_replication/test.py +++ b/tests/integration/test_distributed_ddl_on_cross_replication/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_distributed_ddl_parallel/test.py b/tests/integration/test_distributed_ddl_parallel/test.py index eb98dd3e230..fb14bafc202 100644 --- a/tests/integration/test_distributed_ddl_parallel/test.py +++ b/tests/integration/test_distributed_ddl_parallel/test.py @@ -2,10 +2,12 @@ # pylint: disable=redefined-outer-name # pylint: disable=line-too-long -from functools import wraps import threading import time +from functools import wraps + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_ddl_password/test.py b/tests/integration/test_distributed_ddl_password/test.py index bf2b7979c3c..545fdd3f718 100644 --- a/tests/integration/test_distributed_ddl_password/test.py +++ b/tests/integration/test_distributed_ddl_password/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_distributed_default_database/test.py b/tests/integration/test_distributed_default_database/test.py index 7da9a368997..8b55cbfbbc8 100644 --- a/tests/integration/test_distributed_default_database/test.py +++ b/tests/integration/test_distributed_default_database/test.py @@ -6,14 +6,14 @@ The default database in the distributed table definition is left empty on purpos default database deduction. """ +from contextlib import contextmanager + import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -from contextlib import contextmanager - def bootstrap(cluster): for i, node in enumerate(list(cluster.instances.values())): diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index 43728ff600f..7a843a87ec2 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index 5611f465e8b..18709a8bb3a 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -3,6 +3,7 @@ # pylint: disable=line-too-long import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 9e794555d49..f02d6fb13b9 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -1,7 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION from helpers.client import QueryRuntimeException +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index d74cf97e5c6..a68777cdd24 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -2,9 +2,10 @@ # pylint: disable=redefined-outer-name # pylint: disable=line-too-long -import pytest -import uuid import time +import uuid + +import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index a913c2ebb49..7cd06948c55 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -5,6 +5,7 @@ import uuid import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_over_distributed/test.py b/tests/integration/test_distributed_over_distributed/test.py index c000005e55a..adb4cd21ea5 100644 --- a/tests/integration/test_distributed_over_distributed/test.py +++ b/tests/integration/test_distributed_over_distributed/test.py @@ -3,6 +3,7 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_distributed_respect_user_timeouts/test.py b/tests/integration/test_distributed_respect_user_timeouts/test.py index 906ffb03fc0..dd0953c47dd 100644 --- a/tests/integration/test_distributed_respect_user_timeouts/test.py +++ b/tests/integration/test_distributed_respect_user_timeouts/test.py @@ -1,9 +1,10 @@ import itertools +import logging import os.path import timeit import pytest -import logging + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV diff --git a/tests/integration/test_dns_cache/test.py b/tests/integration/test_dns_cache/test.py index 36401517429..7d652249cab 100644 --- a/tests/integration/test_dns_cache/test.py +++ b/tests/integration/test_dns_cache/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dotnet_client/test.py b/tests/integration/test_dotnet_client/test.py index 2af9b80f720..29ce9bab07c 100644 --- a/tests/integration/test_dotnet_client/test.py +++ b/tests/integration/test_dotnet_client/test.py @@ -1,14 +1,15 @@ # coding: utf-8 import datetime +import logging import math import os import time -import logging import docker import pytest from docker.models.containers import Container + from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_drop_if_empty/test.py b/tests/integration/test_drop_if_empty/test.py index 251ed302b38..36842dbe8c0 100644 --- a/tests/integration/test_drop_if_empty/test.py +++ b/tests/integration/test_drop_if_empty/test.py @@ -3,8 +3,9 @@ import time import pytest import requests -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") diff --git a/tests/integration/test_drop_is_lock_free/test.py b/tests/integration/test_drop_is_lock_free/test.py index 3855bc21f90..32d3e2708d5 100644 --- a/tests/integration/test_drop_is_lock_free/test.py +++ b/tests/integration/test_drop_is_lock_free/test.py @@ -1,11 +1,12 @@ -import time -import pytest import logging +import time from contextlib import contextmanager + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/tests/integration/test_drop_no_local_path/test.py b/tests/integration/test_drop_no_local_path/test.py index 6e587f0a050..bcad8571be6 100644 --- a/tests/integration/test_drop_no_local_path/test.py +++ b/tests/integration/test_drop_no_local_path/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index e0928c6ab08..fc6c225a0d0 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_drop_replica_with_auxiliary_zookeepers/test.py b/tests/integration/test_drop_replica_with_auxiliary_zookeepers/test.py index e21449b93a8..c52190e4239 100644 --- a/tests/integration/test_drop_replica_with_auxiliary_zookeepers/test.py +++ b/tests/integration/test_drop_replica_with_auxiliary_zookeepers/test.py @@ -1,9 +1,10 @@ import time -import helpers.client as client import pytest -from helpers.cluster import ClickHouseCluster + +import helpers.client as client from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 1cf05ff9df4..2a56aafac9f 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index fbf2b59785b..2689ddd1670 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -1,8 +1,9 @@ +import os.path + import pytest -import os.path -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException -import os.path +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_encrypted_disk_replication/test.py b/tests/integration/test_encrypted_disk_replication/test.py index c2aa710ba91..a2ddb50d935 100644 --- a/tests/integration/test_encrypted_disk_replication/test.py +++ b/tests/integration/test_encrypted_disk_replication/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_endpoint_macro_substitution/test.py b/tests/integration/test_endpoint_macro_substitution/test.py index 7dc282a980f..89df23cba97 100644 --- a/tests/integration/test_endpoint_macro_substitution/test.py +++ b/tests/integration/test_endpoint_macro_substitution/test.py @@ -1,7 +1,8 @@ import pytest +from pyhdfs import HdfsClient + from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV -from pyhdfs import HdfsClient disk_types = { "default": "Local", diff --git a/tests/integration/test_executable_dictionary/user_scripts/input.py b/tests/integration/test_executable_dictionary/user_scripts/input.py index 75a3ccac52c..4e389a7e6b4 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py index 11a86737966..586d4313ddd 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py index cbe47041712..fea0296db88 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py index b8297cc42bc..1e8585c71c7 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import re +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py b/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py index 4c131ddffd0..e0d0e4f64fc 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_slow.py b/tests/integration/test_executable_dictionary/user_scripts/input_slow.py index aa8ec0101e2..b0514b46ebd 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_slow.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_slow.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_sum.py b/tests/integration/test_executable_dictionary/user_scripts/input_sum.py index ffdf599c886..4c7428056aa 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_sum.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_sum.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import re +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py index 4c7a03eee80..55294d4d5c0 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import os +import sys if __name__ == "__main__": fd3 = os.fdopen(3) diff --git a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py index 412e7d95299..5494dbe9aad 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import os +import sys if __name__ == "__main__": fd3 = os.fdopen(3) diff --git a/tests/integration/test_executable_table_function/user_scripts/input_signalled.py b/tests/integration/test_executable_table_function/user_scripts/input_signalled.py index fd3ad19039d..8beddcf7108 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_signalled.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py index 79813c2e9c7..46b27f9fc18 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys if __name__ == "__main__": for chunk_header in sys.stdin: diff --git a/tests/integration/test_executable_table_function/user_scripts/input_sum.py b/tests/integration/test_executable_table_function/user_scripts/input_sum.py index b8297cc42bc..1e8585c71c7 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_sum.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_sum.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import re +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py index a04dc9a1b26..b7ac3a89782 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import re +import sys if __name__ == "__main__": for chunk_header in sys.stdin: diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py index 11a86737966..586d4313ddd 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py index cbe47041712..fea0296db88 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 -import sys import os import signal +import sys import time if __name__ == "__main__": diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py index b8297cc42bc..1e8585c71c7 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import re +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py index 955196397d3..4426a4f8628 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import json +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py index 9f3e3c091c2..5a40d38f69a 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import json +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py index 0aad7b1b435..39ae857a6f4 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -import sys import json +import sys if __name__ == "__main__": for line in sys.stdin: diff --git a/tests/integration/test_executable_user_defined_functions_config_reload/test.py b/tests/integration/test_executable_user_defined_functions_config_reload/test.py index 91c93c4593b..ec7c6441551 100644 --- a/tests/integration/test_executable_user_defined_functions_config_reload/test.py +++ b/tests/integration/test_executable_user_defined_functions_config_reload/test.py @@ -1,7 +1,8 @@ +import logging import os import sys import time -import logging + import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tests/integration/test_external_cluster/test.py b/tests/integration/test_external_cluster/test.py index 306ecf66bc7..ac0c0247836 100644 --- a/tests/integration/test_external_cluster/test.py +++ b/tests/integration/test_external_cluster/test.py @@ -1,5 +1,7 @@ import re + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_external_http_authenticator/http_auth_server.py b/tests/integration/test_external_http_authenticator/http_auth_server.py index 6437468c417..d36fd132aa1 100644 --- a/tests/integration/test_external_http_authenticator/http_auth_server.py +++ b/tests/integration/test_external_http_authenticator/http_auth_server.py @@ -2,7 +2,6 @@ import base64 import http.server import json - GOOD_PASSWORD = "good_password" USER_RESPONSES = { "test_user_1": {"settings": {"auth_user": "'test_user'", "auth_num": "UInt64_15"}}, diff --git a/tests/integration/test_external_http_authenticator/test.py b/tests/integration/test_external_http_authenticator/test.py index 286d3a334c1..5edcaa6e166 100644 --- a/tests/integration/test_external_http_authenticator/test.py +++ b/tests/integration/test_external_http_authenticator/test.py @@ -1,14 +1,14 @@ import json import logging -import pytest import os import sys - -from .http_auth_server import GOOD_PASSWORD, USER_RESPONSES +import pytest from helpers.cluster import ClickHouseCluster +from .http_auth_server import GOOD_PASSWORD, USER_RESPONSES + cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( "node", diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py index 3632369154a..dc5f8fa57e7 100644 --- a/tests/integration/test_extreme_deduplication/test.py +++ b/tests/integration/test_extreme_deduplication/test.py @@ -1,8 +1,8 @@ import time import pytest -from helpers.client import CommandRequest -from helpers.client import QueryTimeoutExceedException + +from helpers.client import CommandRequest, QueryTimeoutExceedException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_failed_async_inserts/test.py b/tests/integration/test_failed_async_inserts/test.py index ecb506c36bc..5cc98faead5 100644 --- a/tests/integration/test_failed_async_inserts/test.py +++ b/tests/integration/test_failed_async_inserts/test.py @@ -2,8 +2,6 @@ import logging from time import sleep import pytest -from helpers.cluster import ClickHouseCluster - from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_failed_mutations/test.py b/tests/integration/test_failed_mutations/test.py index c72f5264f03..5a2bf874da2 100644 --- a/tests/integration/test_failed_mutations/test.py +++ b/tests/integration/test_failed_mutations/test.py @@ -1,5 +1,7 @@ import time + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 582748046f9..0058b90973e 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_fetch_partition_should_reset_mutation/test.py b/tests/integration/test_fetch_partition_should_reset_mutation/test.py index 7037393a3d2..10626bb05ed 100644 --- a/tests/integration/test_fetch_partition_should_reset_mutation/test.py +++ b/tests/integration/test_fetch_partition_should_reset_mutation/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_fetch_partition_with_outdated_parts/test.py b/tests/integration/test_fetch_partition_with_outdated_parts/test.py index b78d09b0316..7246082af04 100644 --- a/tests/integration/test_fetch_partition_with_outdated_parts/test.py +++ b/tests/integration/test_fetch_partition_with_outdated_parts/test.py @@ -1,9 +1,10 @@ from __future__ import print_function -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException -import helpers + import pytest +import helpers +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_file_cluster/test.py b/tests/integration/test_file_cluster/test.py index 5d12407e3f2..04ed4c51cee 100644 --- a/tests/integration/test_file_cluster/test.py +++ b/tests/integration/test_file_cluster/test.py @@ -1,12 +1,12 @@ -import logging import csv +import logging import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV - logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) diff --git a/tests/integration/test_file_schema_inference_cache/test.py b/tests/integration/test_file_schema_inference_cache/test.py index dcbae3f2606..cb960904914 100755 --- a/tests/integration/test_file_schema_inference_cache/test.py +++ b/tests/integration/test_file_schema_inference_cache/test.py @@ -1,5 +1,7 @@ -import pytest import time + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_filesystem_cache/test.py b/tests/integration/test_filesystem_cache/test.py index aee8bd25c2e..66acf550191 100644 --- a/tests/integration/test_filesystem_cache/test.py +++ b/tests/integration/test_filesystem_cache/test.py @@ -1,12 +1,13 @@ import logging -import time import os import random +import time import pytest + from helpers.cluster import ClickHouseCluster -from helpers.mock_servers import start_s3_mock, start_mock_servers -from helpers.utility import generate_values, replace_config, SafeThread +from helpers.mock_servers import start_mock_servers, start_s3_mock +from helpers.utility import SafeThread, generate_values, replace_config SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 700e4d05043..66b8f033a54 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -1,7 +1,8 @@ +from pathlib import Path + import pytest from helpers.cluster import ClickHouseCluster -from pathlib import Path cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_force_deduplication/test.py b/tests/integration/test_force_deduplication/test.py index 14c11bc8500..c9fb51c0667 100644 --- a/tests/integration/test_force_deduplication/test.py +++ b/tests/integration/test_force_deduplication/test.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_format_avro_confluent/test.py b/tests/integration/test_format_avro_confluent/test.py index ccaaee83514..b6d49c60c01 100644 --- a/tests/integration/test_format_avro_confluent/test.py +++ b/tests/integration/test_format_avro_confluent/test.py @@ -1,6 +1,7 @@ import io import logging import time +from urllib import parse import avro.schema import pytest @@ -8,8 +9,8 @@ from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, ) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer + from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm -from urllib import parse # Skip on ARM due to Confluent/Kafka if is_arm(): diff --git a/tests/integration/test_format_schema_on_server/test.py b/tests/integration/test_format_schema_on_server/test.py index 3fa10ff1a1e..c3d019ded22 100644 --- a/tests/integration/test_format_schema_on_server/test.py +++ b/tests/integration/test_format_schema_on_server/test.py @@ -1,7 +1,9 @@ -import pytest import os -from helpers.cluster import ClickHouseCluster + +import pytest + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) instance = cluster.add_instance("instance", clickhouse_path_dir="clickhouse_path") diff --git a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py index 50bf0f26075..18d7c6bc3ee 100644 --- a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py +++ b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py index 4a5dfb83f79..d3795600090 100644 --- a/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py +++ b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_graphite_merge_tree/test.py b/tests/integration/test_graphite_merge_tree/test.py index 7c19888df6b..1a55ee28aa1 100644 --- a/tests/integration/test_graphite_merge_tree/test.py +++ b/tests/integration/test_graphite_merge_tree/test.py @@ -3,10 +3,10 @@ import os.path as p import time import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.test_tools import csv_compare +from helpers.test_tools import TSV, csv_compare cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_graphite_merge_tree_typed/test.py b/tests/integration/test_graphite_merge_tree_typed/test.py index 5647489f64f..fb89537c088 100644 --- a/tests/integration/test_graphite_merge_tree_typed/test.py +++ b/tests/integration/test_graphite_merge_tree_typed/test.py @@ -1,13 +1,13 @@ import datetime import os.path as p +import sys import time -import sys import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.test_tools import csv_compare +from helpers.test_tools import TSV, csv_compare cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_group_array_element_size/test.py b/tests/integration/test_group_array_element_size/test.py index 90b2712ffbf..fd0392679e0 100644 --- a/tests/integration/test_group_array_element_size/test.py +++ b/tests/integration/test_group_array_element_size/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 328ba3bc05c..732907eed7a 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -1,21 +1,23 @@ +import gzip import os -import pytest import sys import time -import pytz import uuid -import grpc -from helpers.cluster import ClickHouseCluster, is_arm, run_and_check from threading import Thread -import gzip + +import grpc import lz4.frame +import pytest +import pytz + +from helpers.cluster import ClickHouseCluster, is_arm, run_and_check script_dir = os.path.dirname(os.path.realpath(__file__)) pb2_dir = os.path.join(script_dir, "pb2") if pb2_dir not in sys.path: sys.path.append(pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute pb2/generate.py to generate these modules. - +import clickhouse_grpc_pb2 # Execute pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc GRPC_PORT = 9100 DEFAULT_ENCODING = "utf-8" diff --git a/tests/integration/test_grpc_protocol_ssl/test.py b/tests/integration/test_grpc_protocol_ssl/test.py index 2bc835e9c4c..057c0479608 100644 --- a/tests/integration/test_grpc_protocol_ssl/test.py +++ b/tests/integration/test_grpc_protocol_ssl/test.py @@ -1,15 +1,17 @@ import os -import pytest import sys + import grpc +import pytest + from helpers.cluster import ClickHouseCluster, run_and_check script_dir = os.path.dirname(os.path.realpath(__file__)) pb2_dir = os.path.join(script_dir, "pb2") if pb2_dir not in sys.path: sys.path.append(pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute pb2/generate.py to generate these modules. - +import clickhouse_grpc_pb2 # Execute pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc # The test cluster is configured with certificate for that host name, see 'server-ext.cnf'. # The client have to verify server certificate against that name. Client uses SNI diff --git a/tests/integration/test_host_regexp_hosts_file_resolution/test.py b/tests/integration/test_host_regexp_hosts_file_resolution/test.py index 2c07c4d880f..634b9dbcf6d 100644 --- a/tests/integration/test_host_regexp_hosts_file_resolution/test.py +++ b/tests/integration/test_host_regexp_hosts_file_resolution/test.py @@ -1,7 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check import os +import pytest + +from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check + DOCKER_COMPOSE_PATH = get_docker_compose_path() SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_host_regexp_multiple_ptr_records/test.py b/tests/integration/test_host_regexp_multiple_ptr_records/test.py index 82ae0b6c527..b145fbd0856 100644 --- a/tests/integration/test_host_regexp_multiple_ptr_records/test.py +++ b/tests/integration/test_host_regexp_multiple_ptr_records/test.py @@ -1,8 +1,10 @@ -import pytest -import socket -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check -from time import sleep import os +import socket +from time import sleep + +import pytest + +from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check DOCKER_COMPOSE_PATH = get_docker_compose_path() SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_http_and_readonly/test.py b/tests/integration/test_http_and_readonly/test.py index 1ce3345bf80..34a0752abdb 100644 --- a/tests/integration/test_http_and_readonly/test.py +++ b/tests/integration/test_http_and_readonly/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_http_failover/test.py b/tests/integration/test_http_failover/test.py index 5920fd980ce..758bbf10389 100644 --- a/tests/integration/test_http_failover/test.py +++ b/tests/integration/test_http_failover/test.py @@ -1,10 +1,10 @@ -import pytest from contextlib import nullcontext as does_not_raise -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException -from helpers.test_tools import exec_query_with_retry -from helpers.test_tools import assert_eq_with_retry +import pytest + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry ACCESSIBLE_IPV4 = "10.5.172.10" OTHER_ACCESSIBLE_IPV4 = "10.5.172.20" diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index b2efbf4bb65..efba4f05748 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -1,6 +1,8 @@ import contextlib import os -import urllib.request, urllib.parse, urllib.error +import urllib.error +import urllib.parse +import urllib.request from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_http_native/test.py b/tests/integration/test_http_native/test.py index d55f2a30867..a9e6df85d63 100644 --- a/tests/integration/test_http_native/test.py +++ b/tests/integration/test_http_native/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_https_replication/test.py b/tests/integration/test_https_replication/test.py index 301487aa6cf..b6621675970 100644 --- a/tests/integration/test_https_replication/test.py +++ b/tests/integration/test_https_replication/test.py @@ -3,6 +3,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_https_replication/test_change_ip.py b/tests/integration/test_https_replication/test_change_ip.py index 14fe5351c8d..f9d834cd96a 100644 --- a/tests/integration/test_https_replication/test_change_ip.py +++ b/tests/integration/test_https_replication/test_change_ip.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_https_s3_table_function_with_http_proxy_no_tunneling/test.py b/tests/integration/test_https_s3_table_function_with_http_proxy_no_tunneling/test.py index 3c8a5de8691..cb94e46d452 100644 --- a/tests/integration/test_https_s3_table_function_with_http_proxy_no_tunneling/test.py +++ b/tests/integration/test_https_s3_table_function_with_http_proxy_no_tunneling/test.py @@ -1,9 +1,10 @@ import logging -import helpers.s3_url_proxy_tests_util as proxy_util +import os import pytest + +import helpers.s3_url_proxy_tests_util as proxy_util from helpers.cluster import ClickHouseCluster -import os @pytest.fixture(scope="module") diff --git a/tests/integration/test_insert_distributed_async_send/test.py b/tests/integration/test_insert_distributed_async_send/test.py index 5165c73484c..f8901f524f9 100644 --- a/tests/integration/test_insert_distributed_async_send/test.py +++ b/tests/integration/test_insert_distributed_async_send/test.py @@ -7,8 +7,8 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index 13754d90c72..e16a015029a 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_insert_into_distributed_sync_async/test.py b/tests/integration/test_insert_into_distributed_sync_async/test.py index db9c4405ab9..c5b9608675f 100755 --- a/tests/integration/test_insert_into_distributed_sync_async/test.py +++ b/tests/integration/test_insert_into_distributed_sync_async/test.py @@ -5,9 +5,9 @@ from contextlib import contextmanager import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from helpers.test_tools import TSV -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException, QueryTimeoutExceedException +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py index b1eb0df2d43..46aa0c2dc46 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV diff --git a/tests/integration/test_insert_over_http_query_log/test.py b/tests/integration/test_insert_over_http_query_log/test.py index 6e862e5ddde..f6ba6eaa30a 100644 --- a/tests/integration/test_insert_over_http_query_log/test.py +++ b/tests/integration/test_insert_over_http_query_log/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_inserts_with_keeper_retries/test.py b/tests/integration/test_inserts_with_keeper_retries/test.py index 3294d4ffa02..5c0cdabb106 100644 --- a/tests/integration/test_inserts_with_keeper_retries/test.py +++ b/tests/integration/test_inserts_with_keeper_retries/test.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -import pytest -import time import threading +import time import uuid -from helpers.cluster import ClickHouseCluster from multiprocessing.dummy import Pool -from helpers.network import PartitionManager + +import pytest + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_intersecting_parts/test.py b/tests/integration/test_intersecting_parts/test.py index 3a9732f22de..071178cb956 100644 --- a/tests/integration/test_intersecting_parts/test.py +++ b/tests/integration/test_intersecting_parts/test.py @@ -1,6 +1,7 @@ -import pytest import logging +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_interserver_dns_retires/test.py b/tests/integration/test_interserver_dns_retires/test.py index 7c81f278737..a300a194fe8 100644 --- a/tests/integration/test_interserver_dns_retires/test.py +++ b/tests/integration/test_interserver_dns_retires/test.py @@ -3,12 +3,12 @@ This test makes sure interserver cluster queries handle invalid DNS records for replicas. """ +import multiprocessing.dummy +from contextlib import contextmanager + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from contextlib import contextmanager -import multiprocessing.dummy - def bootstrap(cluster: ClickHouseCluster): node: ClickHouseInstance diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index 8635f5e612a..e75f7684425 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -7,6 +7,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, assert_eq_with_retry diff --git a/tests/integration/test_jbod_ha/test.py b/tests/integration/test_jbod_ha/test.py index 033d751912a..f8507cda97a 100644 --- a/tests/integration/test_jbod_ha/test.py +++ b/tests/integration/test_jbod_ha/test.py @@ -7,6 +7,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_jbod_load_balancing/test.py b/tests/integration/test_jbod_load_balancing/test.py index 204f9740cfd..b638badbcee 100644 --- a/tests/integration/test_jbod_load_balancing/test.py +++ b/tests/integration/test_jbod_load_balancing/test.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_jdbc_bridge/test.py b/tests/integration/test_jdbc_bridge/test.py index 1efd868e4a7..f35368c5ed0 100644 --- a/tests/integration/test_jdbc_bridge/test.py +++ b/tests/integration/test_jdbc_bridge/test.py @@ -1,11 +1,12 @@ import logging import os.path as p -import pytest import uuid +from string import Template + +import pytest from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV -from string import Template cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_jemalloc_percpu_arena/test.py b/tests/integration/test_jemalloc_percpu_arena/test.py index 8de3dcd7ea2..96793a2cc7e 100755 --- a/tests/integration/test_jemalloc_percpu_arena/test.py +++ b/tests/integration/test_jemalloc_percpu_arena/test.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 # pylint: disable=line-too-long +import multiprocessing import os import subprocess -import multiprocessing from tempfile import NamedTemporaryFile -import pytest +import pytest CPU_ID = 4 diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index f0e1480d867..7f91440459d 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -2,6 +2,7 @@ import logging import sys import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index 0446ca5cb47..41c7082637b 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -1,10 +1,11 @@ -import time import logging +import time + import pytest +from kafka import BrokerConnection, KafkaAdminClient, KafkaConsumer, KafkaProducer +from kafka.admin import NewTopic from helpers.cluster import ClickHouseCluster, is_arm -from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection -from kafka.admin import NewTopic if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_keeper_auth/test.py b/tests/integration/test_keeper_auth/test.py index 78fbf84bbe2..8578c966722 100644 --- a/tests/integration/test_keeper_auth/test.py +++ b/tests/integration/test_keeper_auth/test.py @@ -1,15 +1,17 @@ -import pytest import time -from helpers.cluster import ClickHouseCluster -from helpers import keeper_utils + +import pytest from kazoo.client import KazooClient, KazooState -from kazoo.security import ACL, make_digest_acl, make_acl from kazoo.exceptions import ( AuthFailedError, InvalidACLError, - NoAuthError, KazooException, + NoAuthError, ) +from kazoo.security import ACL, make_acl, make_digest_acl + +from helpers import keeper_utils +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_keeper_availability_zone/test.py b/tests/integration/test_keeper_availability_zone/test.py index a2003f8539e..06dc392266a 100644 --- a/tests/integration/test_keeper_availability_zone/test.py +++ b/tests/integration/test_keeper_availability_zone/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.keeper_utils import KeeperClient - cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index b737ac284d2..7e92c298274 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -1,11 +1,13 @@ -import pytest -from helpers.cluster import ClickHouseCluster +import os import random import string -import os import time from multiprocessing.dummy import Pool +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", @@ -362,7 +364,7 @@ def test_multitransactions(started_cluster): assert results[0] == "/test_multitransactions/freddy" assert results[2].startswith("/test_multitransactions/smith0") is True - from kazoo.exceptions import RolledBackError, NoNodeError + from kazoo.exceptions import NoNodeError, RolledBackError for i, zk in enumerate([genuine_zk, fake_zk]): print("Processing ZK", i) diff --git a/tests/integration/test_keeper_broken_logs/test.py b/tests/integration/test_keeper_broken_logs/test.py index 49b8d985ee8..f75e2ae4f20 100644 --- a/tests/integration/test_keeper_broken_logs/test.py +++ b/tests/integration/test_keeper_broken_logs/test.py @@ -1,8 +1,10 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils import time +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index f5f7d855808..be231b4609f 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -1,8 +1,8 @@ import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.keeper_utils import KeeperClient, KeeperException +from helpers.cluster import ClickHouseCluster +from helpers.keeper_utils import KeeperClient, KeeperException +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_disks/test.py b/tests/integration/test_keeper_disks/test.py index 0c91aa03419..2a278d97c64 100644 --- a/tests/integration/test_keeper_disks/test.py +++ b/tests/integration/test_keeper_disks/test.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 +import os + import pytest -from helpers.cluster import ClickHouseCluster, is_arm -import helpers.keeper_utils as keeper_utils from minio.deleteobjects import DeleteObject -import os +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster, is_arm if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_keeper_dynamic_log_level/test.py b/tests/integration/test_keeper_dynamic_log_level/test.py index 5fdeedb6086..db212fc4ca0 100644 --- a/tests/integration/test_keeper_dynamic_log_level/test.py +++ b/tests/integration/test_keeper_dynamic_log_level/test.py @@ -1,6 +1,7 @@ -import pytest -import time import sys +import time + +import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_keeper_feature_flags_config/test.py b/tests/integration/test_keeper_feature_flags_config/test.py index 5bbd0d668c6..c5817736e87 100644 --- a/tests/integration/test_keeper_feature_flags_config/test.py +++ b/tests/integration/test_keeper_feature_flags_config/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest import os -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils + +import pytest from kazoo.client import KazooClient, KazooState +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_force_recovery/test.py b/tests/integration/test_keeper_force_recovery/test.py index f630e5a422b..a282e77e3cd 100644 --- a/tests/integration/test_keeper_force_recovery/test.py +++ b/tests/integration/test_keeper_force_recovery/test.py @@ -1,13 +1,13 @@ import os -import pytest import socket -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils import time - +import pytest from kazoo.client import KazooClient, KazooRetry +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + CLUSTER_SIZE = 5 QUORUM_SIZE = CLUSTER_SIZE // 2 + 1 diff --git a/tests/integration/test_keeper_force_recovery_single_node/test.py b/tests/integration/test_keeper_force_recovery_single_node/test.py index 132c5488df6..d142a117be2 100644 --- a/tests/integration/test_keeper_force_recovery_single_node/test.py +++ b/tests/integration/test_keeper_force_recovery_single_node/test.py @@ -1,13 +1,13 @@ import os -import pytest import socket -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils import time - +import pytest from kazoo.client import KazooClient, KazooRetry +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + CLUSTER_SIZE = 3 cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index 83503122729..6d6f3c9a8ad 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -1,9 +1,11 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils -import time import csv import re +import time + +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_keeper_four_word_command/test_allow_list.py b/tests/integration/test_keeper_four_word_command/test_allow_list.py index 4bf8ae1ab53..ad57cf5f61e 100644 --- a/tests/integration/test_keeper_four_word_command/test_allow_list.py +++ b/tests/integration/test_keeper_four_word_command/test_allow_list.py @@ -1,8 +1,10 @@ import socket -import pytest -from helpers.cluster import ClickHouseCluster import time +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", main_configs=["configs/keeper_config_with_allow_list.xml"], stay_alive=True diff --git a/tests/integration/test_keeper_http_control/test.py b/tests/integration/test_keeper_http_control/test.py index 65dc5bea909..fed77c64536 100644 --- a/tests/integration/test_keeper_http_control/test.py +++ b/tests/integration/test_keeper_http_control/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import os + import pytest import requests +import helpers.keeper_utils as keeper_utils from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -import helpers.keeper_utils as keeper_utils cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_incorrect_config/test.py b/tests/integration/test_keeper_incorrect_config/test.py index 95482745b31..e1f93f3c27e 100644 --- a/tests/integration/test_keeper_incorrect_config/test.py +++ b/tests/integration/test_keeper_incorrect_config/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index 8cab03b6e2d..5be0a12ddbf 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as ku -from multiprocessing.dummy import Pool import os +from multiprocessing.dummy import Pool + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_map_retries/test.py b/tests/integration/test_keeper_map_retries/test.py index c6760e5d1a2..8b113a5f9e2 100644 --- a/tests/integration/test_keeper_map_retries/test.py +++ b/tests/integration/test_keeper_map_retries/test.py @@ -1,9 +1,9 @@ +import os + import pytest from helpers.cluster import ClickHouseCluster -import os - CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_memory_soft_limit/test.py b/tests/integration/test_keeper_memory_soft_limit/test.py index d6f3d013a7b..e4f8261b05f 100644 --- a/tests/integration/test_keeper_memory_soft_limit/test.py +++ b/tests/integration/test_keeper_memory_soft_limit/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 import random import string + import pytest -from helpers.cluster import ClickHouseCluster -from helpers import keeper_utils from kazoo.client import KazooClient, KazooState from kazoo.exceptions import ConnectionLoss +from helpers import keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__, keeper_config_dir="configs/") # clickhouse itself will use external zookeeper diff --git a/tests/integration/test_keeper_mntr_data_size/test.py b/tests/integration/test_keeper_mntr_data_size/test.py index 847b3e87ae3..26c1a83adb6 100644 --- a/tests/integration/test_keeper_mntr_data_size/test.py +++ b/tests/integration/test_keeper_mntr_data_size/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils import random import string + +import pytest from kazoo.client import KazooClient, KazooState +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_mntr_pressure/test.py b/tests/integration/test_keeper_mntr_pressure/test.py index d351b238ead..4dd981b1979 100644 --- a/tests/integration/test_keeper_mntr_pressure/test.py +++ b/tests/integration/test_keeper_mntr_pressure/test.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils -import pytest -import random -import string import os +import random +import socket +import string +import threading import time from io import StringIO -import socket -import threading +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_multinode_simple/test.py b/tests/integration/test_keeper_multinode_simple/test.py index 1999f361dd4..085e72a091b 100644 --- a/tests/integration/test_keeper_multinode_simple/test.py +++ b/tests/integration/test_keeper_multinode_simple/test.py @@ -1,11 +1,13 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time from multiprocessing.dummy import Pool + +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_keeper_nodes_add/test.py b/tests/integration/test_keeper_nodes_add/test.py index c5282de1bc8..90a70c9ef42 100644 --- a/tests/integration/test_keeper_nodes_add/test.py +++ b/tests/integration/test_keeper_nodes_add/test.py @@ -1,16 +1,18 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time from multiprocessing.dummy import Pool -from helpers.test_tools import assert_eq_with_retry + +import pytest from kazoo.client import KazooClient, KazooState +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_nodes_move/test.py b/tests/integration/test_keeper_nodes_move/test.py index 8ac7bc9b5e2..e43aeeae72c 100644 --- a/tests/integration/test_keeper_nodes_move/test.py +++ b/tests/integration/test_keeper_nodes_move/test.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster import os import time from multiprocessing.dummy import Pool -from helpers.test_tools import assert_eq_with_retry -import helpers.keeper_utils as keeper_utils + +import pytest from kazoo.client import KazooClient, KazooState +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_nodes_remove/test.py b/tests/integration/test_keeper_nodes_remove/test.py index 12c37e54927..314628ce74e 100644 --- a/tests/integration/test_keeper_nodes_remove/test.py +++ b/tests/integration/test_keeper_nodes_remove/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import time import os +import time + +import pytest from kazoo.client import KazooClient, KazooState +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_persistent_log/test.py b/tests/integration/test_keeper_persistent_log/test.py index 4164ffb33d3..5051e0759ce 100644 --- a/tests/integration/test_keeper_persistent_log/test.py +++ b/tests/integration/test_keeper_persistent_log/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster +import os import random import string -import os import time + +import pytest from kazoo.client import KazooClient, KazooState +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_persistent_log_multinode/test.py b/tests/integration/test_keeper_persistent_log_multinode/test.py index 1552abd32e9..b055d2806e7 100644 --- a/tests/integration/test_keeper_persistent_log_multinode/test.py +++ b/tests/integration/test_keeper_persistent_log_multinode/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_profiler/test.py b/tests/integration/test_keeper_profiler/test.py index 848929df086..08d786e86bf 100644 --- a/tests/integration/test_keeper_profiler/test.py +++ b/tests/integration/test_keeper_profiler/test.py @@ -1,8 +1,8 @@ import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.keeper_utils import KeeperClient, KeeperException +from helpers.cluster import ClickHouseCluster +from helpers.keeper_utils import KeeperClient, KeeperException +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_reconfig_add/test.py b/tests/integration/test_keeper_reconfig_add/test.py index 724bfdef492..c26817f4cb7 100644 --- a/tests/integration/test_keeper_reconfig_add/test.py +++ b/tests/integration/test_keeper_reconfig_add/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -import helpers.keeper_utils as ku import os import typing as tp +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster, ClickHouseInstance + cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_reconfig_remove/test.py b/tests/integration/test_keeper_reconfig_remove/test.py index d23d771edba..79f8dcf52e7 100644 --- a/tests/integration/test_keeper_reconfig_remove/test.py +++ b/tests/integration/test_keeper_reconfig_remove/test.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 +import os import subprocess +import typing as tp import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance + import helpers.keeper_utils as ku -import os -import typing as tp +from helpers.cluster import ClickHouseCluster, ClickHouseInstance cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_reconfig_remove_many/test.py b/tests/integration/test_keeper_reconfig_remove_many/test.py index 0f9d2fce374..afb06766dbe 100644 --- a/tests/integration/test_keeper_reconfig_remove_many/test.py +++ b/tests/integration/test_keeper_reconfig_remove_many/test.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -import helpers.keeper_utils as ku import os import typing as tp +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster, ClickHouseInstance cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_reconfig_replace_leader/test.py b/tests/integration/test_keeper_reconfig_replace_leader/test.py index 8e621eef279..fa88bacd809 100644 --- a/tests/integration/test_keeper_reconfig_replace_leader/test.py +++ b/tests/integration/test_keeper_reconfig_replace_leader/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from os.path import join, dirname, realpath import time -import helpers.keeper_utils as ku import typing as tp +from os.path import dirname, join, realpath + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster, ClickHouseInstance cluster = ClickHouseCluster(__file__) CONFIG_DIR = join(dirname(realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py b/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py index 95639edf2d0..68ea4a5f080 100644 --- a/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py +++ b/tests/integration/test_keeper_reconfig_replace_leader_in_one_command/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from os.path import join, dirname, realpath import time -import helpers.keeper_utils as ku import typing as tp +from os.path import dirname, join, realpath + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster, ClickHouseInstance cluster = ClickHouseCluster(__file__) CONFIG_DIR = join(dirname(realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_restore_from_snapshot/test.py b/tests/integration/test_keeper_restore_from_snapshot/test.py index e4d5793bb17..1a0baa0f99a 100644 --- a/tests/integration/test_keeper_restore_from_snapshot/test.py +++ b/tests/integration/test_keeper_restore_from_snapshot/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_restore_from_snapshot/test_disk_s3.py b/tests/integration/test_keeper_restore_from_snapshot/test_disk_s3.py index 1226df75203..e0c7bc99e9d 100644 --- a/tests/integration/test_keeper_restore_from_snapshot/test_disk_s3.py +++ b/tests/integration/test_keeper_restore_from_snapshot/test_disk_s3.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_s3_snapshot/test.py b/tests/integration/test_keeper_s3_snapshot/test.py index b6c25305aef..a7f99bf5bee 100644 --- a/tests/integration/test_keeper_s3_snapshot/test.py +++ b/tests/integration/test_keeper_s3_snapshot/test.py @@ -1,12 +1,13 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from time import sleep -from retry import retry from multiprocessing.dummy import Pool -import helpers.keeper_utils as keeper_utils -from minio.deleteobjects import DeleteObject +from time import sleep +import pytest from kazoo.client import KazooClient +from minio.deleteobjects import DeleteObject +from retry import retry + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster # from kazoo.protocol.serialization import Connect, read_buffer, write_buffer diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index 2a17afac75b..00bc72658c6 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import string import os +import string import time +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index cd012ad6e9e..aa8882d55e5 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -1,13 +1,14 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils -import time import socket import struct +import time +import pytest from kazoo.client import KazooClient from kazoo.exceptions import NoNodeError +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + # from kazoo.protocol.serialization import Connect, read_buffer, write_buffer cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_snapshot_on_exit/test.py b/tests/integration/test_keeper_snapshot_on_exit/test.py index 327affc8372..f73a683b79a 100644 --- a/tests/integration/test_keeper_snapshot_on_exit/test.py +++ b/tests/integration/test_keeper_snapshot_on_exit/test.py @@ -1,8 +1,10 @@ -import pytest -from helpers.cluster import ClickHouseCluster import os + +import pytest from kazoo.client import KazooClient +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_keeper_snapshot_small_distance/test.py b/tests/integration/test_keeper_snapshot_small_distance/test.py index 612c5b3c65d..3f8c8c7ae04 100644 --- a/tests/integration/test_keeper_snapshot_small_distance/test.py +++ b/tests/integration/test_keeper_snapshot_small_distance/test.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils -from multiprocessing.dummy import Pool -from kazoo.client import KazooClient, KazooRetry -from kazoo.handlers.threading import KazooTimeoutError +import os import random import string -import os import time +from multiprocessing.dummy import Pool + +import pytest +from kazoo.client import KazooClient, KazooRetry +from kazoo.handlers.threading import KazooTimeoutError + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_keeper_snapshots/test.py b/tests/integration/test_keeper_snapshots/test.py index fa472367bd7..1af3318e24b 100644 --- a/tests/integration/test_keeper_snapshots/test.py +++ b/tests/integration/test_keeper_snapshots/test.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os + +import pytest from kazoo.client import KazooClient +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_snapshots_multinode/test.py b/tests/integration/test_keeper_snapshots_multinode/test.py index a68a34dae2e..27a2d94b488 100644 --- a/tests/integration/test_keeper_snapshots_multinode/test.py +++ b/tests/integration/test_keeper_snapshots_multinode/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True diff --git a/tests/integration/test_keeper_three_nodes_start/test.py b/tests/integration/test_keeper_three_nodes_start/test.py index 6576d386fcb..7806444a00d 100644 --- a/tests/integration/test_keeper_three_nodes_start/test.py +++ b/tests/integration/test_keeper_three_nodes_start/test.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster +import os import random import string -import os import time from multiprocessing.dummy import Pool -from helpers.test_tools import assert_eq_with_retry + +import pytest from kazoo.client import KazooClient, KazooState +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True diff --git a/tests/integration/test_keeper_three_nodes_two_alive/test.py b/tests/integration/test_keeper_three_nodes_two_alive/test.py index 1b57bf602df..47cdb6bd8ff 100644 --- a/tests/integration/test_keeper_three_nodes_two_alive/test.py +++ b/tests/integration/test_keeper_three_nodes_two_alive/test.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time from multiprocessing.dummy import Pool -from helpers.test_tools import assert_eq_with_retry + +import pytest from kazoo.client import KazooClient, KazooState +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_keeper_two_nodes_cluster/test.py b/tests/integration/test_keeper_two_nodes_cluster/test.py index c6bc0ebd33a..da7499d21ae 100644 --- a/tests/integration/test_keeper_two_nodes_cluster/test.py +++ b/tests/integration/test_keeper_two_nodes_cluster/test.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time from multiprocessing.dummy import Pool + +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_keeper_znode_time/test.py b/tests/integration/test_keeper_znode_time/test.py index f2076acc4d2..abede05afcc 100644 --- a/tests/integration/test_keeper_znode_time/test.py +++ b/tests/integration/test_keeper_znode_time/test.py @@ -1,11 +1,13 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils +import os import random import string -import os import time from multiprocessing.dummy import Pool + +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index de5a9416119..5bc84f9e072 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster -import helpers.keeper_utils as keeper_utils -from kazoo.client import KazooClient -from kazoo.retry import KazooRetry -from kazoo.security import make_acl -from kazoo.handlers.threading import KazooTimeoutError import os import time +import pytest +from kazoo.client import KazooClient +from kazoo.handlers.threading import KazooTimeoutError +from kazoo.retry import KazooRetry +from kazoo.security import make_acl + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_kerberos_auth/test.py b/tests/integration/test_kerberos_auth/test.py index 2d0dce33c14..babbdf4dba8 100644 --- a/tests/integration/test_kerberos_auth/test.py +++ b/tests/integration/test_kerberos_auth/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster, is_arm if is_arm(): diff --git a/tests/integration/test_lazy_database/test.py b/tests/integration/test_lazy_database/test.py index 6890aa87374..a01d75c75ba 100644 --- a/tests/integration/test_lazy_database/test.py +++ b/tests/integration/test_lazy_database/test.py @@ -1,7 +1,9 @@ import logging -import time -import pytest import os +import time + +import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_ldap_external_user_directory/test.py b/tests/integration/test_ldap_external_user_directory/test.py index 277a17eed7b..6c25c0ac789 100644 --- a/tests/integration/test_ldap_external_user_directory/test.py +++ b/tests/integration/test_ldap_external_user_directory/test.py @@ -1,5 +1,7 @@ import logging + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index a4dca545d44..6254735a18f 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -1,8 +1,9 @@ +import logging import os import os.path as p -import pytest import time -import logging + +import pytest from helpers.cluster import ClickHouseCluster, run_and_check diff --git a/tests/integration/test_library_bridge/test_exiled.py b/tests/integration/test_library_bridge/test_exiled.py index 56be1bec20a..46a6ac1eaa4 100644 --- a/tests/integration/test_library_bridge/test_exiled.py +++ b/tests/integration/test_library_bridge/test_exiled.py @@ -1,8 +1,9 @@ +import logging import os import os.path as p -import pytest import time -import logging + +import pytest from helpers.cluster import ClickHouseCluster, run_and_check from test_library_bridge.test import create_dict_simple diff --git a/tests/integration/test_limit_materialized_view_count/test.py b/tests/integration/test_limit_materialized_view_count/test.py index c14c5b2055e..d355b29ebe5 100644 --- a/tests/integration/test_limit_materialized_view_count/test.py +++ b/tests/integration/test_limit_materialized_view_count/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_limited_replicated_fetches/test.py b/tests/integration/test_limited_replicated_fetches/test.py index bec575df7cd..83577a624e4 100644 --- a/tests/integration/test_limited_replicated_fetches/test.py +++ b/tests/integration/test_limited_replicated_fetches/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -import pytest -import time -from helpers.cluster import ClickHouseCluster -from helpers.network import PartitionManager +import os import random import string -import os +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_log_family_hdfs/test.py b/tests/integration/test_log_family_hdfs/test.py index 6c3d28d2e3c..2a825006476 100644 --- a/tests/integration/test_log_family_hdfs/test.py +++ b/tests/integration/test_log_family_hdfs/test.py @@ -2,10 +2,10 @@ import logging import sys import pytest -from helpers.cluster import ClickHouseCluster, is_arm - from pyhdfs import HdfsClient +from helpers.cluster import ClickHouseCluster, is_arm + if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_log_family_s3/test.py b/tests/integration/test_log_family_s3/test.py index ed84bdf48e6..b123ee4335b 100644 --- a/tests/integration/test_log_family_s3/test.py +++ b/tests/integration/test_log_family_s3/test.py @@ -2,6 +2,7 @@ import logging import sys import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_log_levels_update/test.py b/tests/integration/test_log_levels_update/test.py index 4b83b6431fc..c675e0565f2 100644 --- a/tests/integration/test_log_levels_update/test.py +++ b/tests/integration/test_log_levels_update/test.py @@ -1,6 +1,7 @@ -import pytest import re +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_log_lz4_streaming/test.py b/tests/integration/test_log_lz4_streaming/test.py index 05c0c809b5a..5433113561d 100644 --- a/tests/integration/test_log_lz4_streaming/test.py +++ b/tests/integration/test_log_lz4_streaming/test.py @@ -1,6 +1,7 @@ -import pytest import time +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index c2469d6c524..dd9a6a65ba9 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import pytest -import time import ast import random +import time + +import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_lost_part_during_startup/test.py b/tests/integration/test_lost_part_during_startup/test.py index a013ec5d48d..ff7c73963c4 100644 --- a/tests/integration/test_lost_part_during_startup/test.py +++ b/tests/integration/test_lost_part_during_startup/test.py @@ -2,6 +2,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_manipulate_statistics/test.py b/tests/integration/test_manipulate_statistics/test.py index 3a1c5ad5b96..0fc9f0a0092 100644 --- a/tests/integration/test_manipulate_statistics/test.py +++ b/tests/integration/test_manipulate_statistics/test.py @@ -1,6 +1,7 @@ -import pytest import logging +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 797986bbf01..d6a0cfb282e 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -1,6 +1,9 @@ -import pytest -import random, string +import random import re +import string + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_match_process_uid_against_data_owner/test.py b/tests/integration/test_match_process_uid_against_data_owner/test.py index 65e4e902853..71cfd39565c 100644 --- a/tests/integration/test_match_process_uid_against_data_owner/test.py +++ b/tests/integration/test_match_process_uid_against_data_owner/test.py @@ -1,7 +1,9 @@ import os import pwd import re + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_materialized_mysql_database/materialized_with_ddl.py b/tests/integration/test_materialized_mysql_database/materialized_with_ddl.py index 9a99f0c9aa8..56a8be1dcf1 100644 --- a/tests/integration/test_materialized_mysql_database/materialized_with_ddl.py +++ b/tests/integration/test_materialized_mysql_database/materialized_with_ddl.py @@ -1,15 +1,15 @@ +import logging +import random +import threading import time +from multiprocessing.dummy import Pool import pymysql.cursors import pytest -from helpers.network import PartitionManager -import logging + from helpers.client import QueryRuntimeException from helpers.cluster import get_docker_compose_path, run_and_check -import random - -import threading -from multiprocessing.dummy import Pool +from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 5efef3624db..e8816de2034 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -1,14 +1,16 @@ +import logging import os import time + import pymysql.cursors import pytest + from helpers.cluster import ( ClickHouseCluster, ClickHouseInstance, get_docker_compose_path, is_arm, ) -import logging from . import materialized_with_ddl diff --git a/tests/integration/test_materialized_view_restart_server/test.py b/tests/integration/test_materialized_view_restart_server/test.py index bfadec5136b..af06589642d 100755 --- a/tests/integration/test_materialized_view_restart_server/test.py +++ b/tests/integration/test_materialized_view_restart_server/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py index 0142c7db746..7b50d8bd086 100644 --- a/tests/integration/test_max_authentication_methods_per_user/test.py +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -1,6 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_max_rows_to_read_leaf_with_view/test.py b/tests/integration/test_max_rows_to_read_leaf_with_view/test.py index d5fe9c97aca..2d5aff96c80 100755 --- a/tests/integration/test_max_rows_to_read_leaf_with_view/test.py +++ b/tests/integration/test_max_rows_to_read_leaf_with_view/test.py @@ -1,8 +1,9 @@ from contextlib import contextmanager import pytest -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_max_suspicious_broken_parts_replicated/test.py b/tests/integration/test_max_suspicious_broken_parts_replicated/test.py index 683715da404..976bb7d3e7b 100644 --- a/tests/integration/test_max_suspicious_broken_parts_replicated/test.py +++ b/tests/integration/test_max_suspicious_broken_parts_replicated/test.py @@ -4,9 +4,10 @@ # pylint: disable=redefined-outer-name # pylint: disable=line-too-long +import os + import pytest -import os from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py index db68a38c1b1..1d395986f01 100644 --- a/tests/integration/test_memory_limit/test.py +++ b/tests/integration/test_memory_limit/test.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import logging import time -import pytest -from helpers.cluster import ClickHouseCluster from multiprocessing.dummy import Pool +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_memory_limit_observer/test.py b/tests/integration/test_memory_limit_observer/test.py index 0eda165b1d2..d23d8a933fc 100644 --- a/tests/integration/test_memory_limit_observer/test.py +++ b/tests/integration/test_memory_limit_observer/test.py @@ -1,7 +1,8 @@ -import pytest import logging import time +import pytest + from helpers.cluster import ClickHouseCluster, run_and_check cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_memory_profiler_min_max_borders/test.py b/tests/integration/test_memory_profiler_min_max_borders/test.py index df9439bc2bb..dec8cf0d791 100644 --- a/tests/integration/test_memory_profiler_min_max_borders/test.py +++ b/tests/integration/test_memory_profiler_min_max_borders/test.py @@ -1,6 +1,7 @@ -from helpers.cluster import ClickHouseCluster import pytest +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", diff --git a/tests/integration/test_merge_table_over_distributed/test.py b/tests/integration/test_merge_table_over_distributed/test.py index 486fea8ba8d..ee74a242eac 100644 --- a/tests/integration/test_merge_table_over_distributed/test.py +++ b/tests/integration/test_merge_table_over_distributed/test.py @@ -1,6 +1,7 @@ from contextlib import contextmanager import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 45ae88f427e..3ed05ffd30c 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -1,15 +1,14 @@ import logging -import time import os +import time import pytest +from azure.storage.blob import BlobServiceClient from helpers.cluster import ClickHouseCluster -from helpers.utility import generate_values, replace_config, SafeThread -from azure.storage.blob import BlobServiceClient +from helpers.utility import SafeThread, generate_values, replace_config from test_storage_azure_blob_storage.test import azure_query - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) NODE_NAME = "node" TABLE_NAME = "blob_storage_table" diff --git a/tests/integration/test_merge_tree_check_part_with_cache/test.py b/tests/integration/test_merge_tree_check_part_with_cache/test.py index 1f50a5ab9de..343ab16dfde 100644 --- a/tests/integration/test_merge_tree_check_part_with_cache/test.py +++ b/tests/integration/test_merge_tree_check_part_with_cache/test.py @@ -1,6 +1,8 @@ -import pytest -import os import json +import os + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py index c6a96f3ed1b..1d1d08d0620 100644 --- a/tests/integration/test_merge_tree_empty_parts/test.py +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -1,9 +1,9 @@ import pytest + import helpers.client import helpers.cluster from helpers.test_tools import assert_eq_with_retry - cluster = helpers.cluster.ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 5ca7dc5feb0..6ab605ab857 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -1,14 +1,16 @@ import logging -import time import os +import time import pytest +from pyhdfs import HdfsClient + from helpers.cluster import ClickHouseCluster, is_arm from helpers.utility import generate_values -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts - -from pyhdfs import HdfsClient +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) CONFIG_PATH = os.path.join( diff --git a/tests/integration/test_merge_tree_load_marks/test.py b/tests/integration/test_merge_tree_load_marks/test.py index a7078017ac9..b283b4acec1 100644 --- a/tests/integration/test_merge_tree_load_marks/test.py +++ b/tests/integration/test_merge_tree_load_marks/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py index 5d575fa95f5..ebd68b70bf1 100644 --- a/tests/integration/test_merge_tree_load_parts/test.py +++ b/tests/integration/test_merge_tree_load_parts/test.py @@ -1,10 +1,11 @@ +import time + import pytest + import helpers.client import helpers.cluster -import time from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk - cluster = helpers.cluster.ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_merge_tree_s3/s3_mocks/no_delete_objects.py b/tests/integration/test_merge_tree_s3/s3_mocks/no_delete_objects.py index 111f3a490c2..68c1f43f13d 100644 --- a/tests/integration/test_merge_tree_s3/s3_mocks/no_delete_objects.py +++ b/tests/integration/test_merge_tree_s3/s3_mocks/no_delete_objects.py @@ -5,7 +5,6 @@ import socketserver import sys import urllib.parse - UPSTREAM_HOST = "minio1:9001" random.seed("No delete objects/1.0") diff --git a/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py b/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py index c4a1e5ea1f5..6ccbf9b105c 100644 --- a/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py +++ b/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py @@ -5,7 +5,6 @@ import socketserver import sys import urllib.parse - UPSTREAM_HOST = "minio1:9001" random.seed("Unstable proxy/1.0") diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index e90fafa8461..baca7b2c186 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -1,15 +1,18 @@ import logging -import time import os +import time import uuid import pytest + from helpers.cluster import ClickHouseCluster -from helpers.mock_servers import start_s3_mock, start_mock_servers -from helpers.utility import generate_values, replace_config, SafeThread -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts -from helpers.wait_for_helpers import wait_for_merges +from helpers.mock_servers import start_mock_servers, start_s3_mock +from helpers.utility import SafeThread, generate_values, replace_config +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, + wait_for_merges, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 4613fdb850b..ea7fe110684 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -1,6 +1,6 @@ -from bottle import request, route, run, response from threading import Lock +from bottle import request, response, route, run # Endpoint can be configured to throw 500 error on N-th request attempt. # In usual situation just redirects to original Minio server. diff --git a/tests/integration/test_merge_tree_s3_with_cache/test.py b/tests/integration/test_merge_tree_s3_with_cache/test.py index 067ed4f9679..e48fd8ff391 100644 --- a/tests/integration/test_merge_tree_s3_with_cache/test.py +++ b/tests/integration/test_merge_tree_s3_with_cache/test.py @@ -1,6 +1,7 @@ import logging import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_merge_tree_settings_constraints/test.py b/tests/integration/test_merge_tree_settings_constraints/test.py index be6e2a31873..9d1e1ede956 100644 --- a/tests/integration/test_merge_tree_settings_constraints/test.py +++ b/tests/integration/test_merge_tree_settings_constraints/test.py @@ -1,10 +1,12 @@ -import pytest import asyncio -import re -import random import os.path +import random +import re + +import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) instance = cluster.add_instance("instance", user_configs=["users.xml"]) diff --git a/tests/integration/test_modify_engine_on_restart/test.py b/tests/integration/test_modify_engine_on_restart/test.py index 289b25dd89e..e819b229cd8 100644 --- a/tests/integration/test_modify_engine_on_restart/test.py +++ b/tests/integration/test_modify_engine_on_restart/test.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_args.py b/tests/integration/test_modify_engine_on_restart/test_args.py index f83d540bfb9..0d17e5905b3 100644 --- a/tests/integration/test_modify_engine_on_restart/test_args.py +++ b/tests/integration/test_modify_engine_on_restart/test_args.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_mv.py b/tests/integration/test_modify_engine_on_restart/test_mv.py index 30cb2ddc5e5..7d6df5b201c 100644 --- a/tests/integration/test_modify_engine_on_restart/test_mv.py +++ b/tests/integration/test_modify_engine_on_restart/test_mv.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_ordinary.py b/tests/integration/test_modify_engine_on_restart/test_ordinary.py index fd86417a216..826abb8c88d 100644 --- a/tests/integration/test_modify_engine_on_restart/test_ordinary.py +++ b/tests/integration/test_modify_engine_on_restart/test_ordinary.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_storage_policies.py b/tests/integration/test_modify_engine_on_restart/test_storage_policies.py index e49af164ed7..f200d31fef1 100644 --- a/tests/integration/test_modify_engine_on_restart/test_storage_policies.py +++ b/tests/integration/test_modify_engine_on_restart/test_storage_policies.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_unusual_path.py b/tests/integration/test_modify_engine_on_restart/test_unusual_path.py index e82f48e8b34..d55106e44bb 100644 --- a/tests/integration/test_modify_engine_on_restart/test_unusual_path.py +++ b/tests/integration/test_modify_engine_on_restart/test_unusual_path.py @@ -1,6 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_modify_engine_on_restart/test_zk_path_exists.py b/tests/integration/test_modify_engine_on_restart/test_zk_path_exists.py index 3bf492cf69d..f79bd48ba7d 100644 --- a/tests/integration/test_modify_engine_on_restart/test_zk_path_exists.py +++ b/tests/integration/test_modify_engine_on_restart/test_zk_path_exists.py @@ -1,9 +1,7 @@ import pytest -from test_modify_engine_on_restart.common import ( - get_table_path, - set_convert_flags, -) + from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import get_table_path, set_convert_flags cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/test.py b/tests/integration/test_move_partition_to_disk_on_cluster/test.py index c639e080cdf..9614bc283d7 100644 --- a/tests/integration/test_move_partition_to_disk_on_cluster/test.py +++ b/tests/integration/test_move_partition_to_disk_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_move_partition_to_volume_async/test.py b/tests/integration/test_move_partition_to_volume_async/test.py index cdd2ee126c0..85021955cbe 100644 --- a/tests/integration/test_move_partition_to_volume_async/test.py +++ b/tests/integration/test_move_partition_to_volume_async/test.py @@ -1,16 +1,18 @@ #!/usr/bin/env python3 import logging -import time import os +import time import pytest -from helpers.cluster import ClickHouseCluster -from helpers.mock_servers import start_s3_mock, start_mock_servers -from helpers.utility import generate_values, replace_config, SafeThread -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts -from helpers.wait_for_helpers import wait_for_merges +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers, start_s3_mock +from helpers.utility import SafeThread, generate_values, replace_config +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, + wait_for_merges, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_move_ttl_broken_compatibility/test.py b/tests/integration/test_move_ttl_broken_compatibility/test.py index f9eab8b5ebb..a19fc036c35 100644 --- a/tests/integration/test_move_ttl_broken_compatibility/test.py +++ b/tests/integration/test_move_ttl_broken_compatibility/test.py @@ -5,10 +5,10 @@ import random import string import time -import pytest -from helpers.cluster import ClickHouseCluster import minio +import pytest +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_multi_access_storage_role_management/test.py b/tests/integration/test_multi_access_storage_role_management/test.py index 77aea411394..275848dd895 100644 --- a/tests/integration/test_multi_access_storage_role_management/test.py +++ b/tests/integration/test_multi_access_storage_role_management/test.py @@ -1,8 +1,10 @@ -import pytest import os + +import pytest + +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index e97ffeb4cc3..478435dda61 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -7,6 +7,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_mutations_hardlinks/test.py b/tests/integration/test_mutations_hardlinks/test.py index f70cbccefa5..c3f9c739367 100644 --- a/tests/integration/test_mutations_hardlinks/test.py +++ b/tests/integration/test_mutations_hardlinks/test.py @@ -3,6 +3,7 @@ import time from multiprocessing.dummy import Pool import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py index 17f2a27e49b..ceb964c8339 100644 --- a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py @@ -1,8 +1,8 @@ import pytest + import helpers.client import helpers.cluster - cluster = helpers.cluster.ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index 0b4a750b38e..23028bff214 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_mutations_with_projection/test.py b/tests/integration/test_mutations_with_projection/test.py index da34764067f..fa03064d49b 100644 --- a/tests/integration/test_mutations_with_projection/test.py +++ b/tests/integration/test_mutations_with_projection/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_mysql57_database_engine/test.py b/tests/integration/test_mysql57_database_engine/test.py index 410d277d667..67aa98af2c3 100644 --- a/tests/integration/test_mysql57_database_engine/test.py +++ b/tests/integration/test_mysql57_database_engine/test.py @@ -4,11 +4,11 @@ from string import Template import pymysql.cursors import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager - if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_mysql_database_engine/test.py b/tests/integration/test_mysql_database_engine/test.py index 44c23374b3a..fcb3661c466 100644 --- a/tests/integration/test_mysql_database_engine/test.py +++ b/tests/integration/test_mysql_database_engine/test.py @@ -4,6 +4,7 @@ from string import Template import pymysql.cursors import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index 094ae7b9fd0..e84396ad300 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -2,21 +2,17 @@ import datetime import fnmatch +import logging import math import os import time - -import logging from typing import Literal import docker import pymysql.connections import pytest -from helpers.cluster import ( - ClickHouseCluster, - get_docker_compose_path, - run_and_check, -) + +from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 32846c79d23..ed80898ebc7 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -1,10 +1,12 @@ import logging -import pytest import os import time -from helpers.cluster import ClickHouseCluster from contextlib import nullcontext as does_not_raise + +import pytest + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) NAMED_COLLECTIONS_CONFIG = os.path.join( diff --git a/tests/integration/test_named_collections_encrypted/test.py b/tests/integration/test_named_collections_encrypted/test.py index 7dff32fa6c9..982c456b151 100644 --- a/tests/integration/test_named_collections_encrypted/test.py +++ b/tests/integration/test_named_collections_encrypted/test.py @@ -1,6 +1,8 @@ import logging -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_named_collections_if_exists_on_cluster/test.py b/tests/integration/test_named_collections_if_exists_on_cluster/test.py index 5f5be9156b9..5cf57ba09ec 100644 --- a/tests/integration/test_named_collections_if_exists_on_cluster/test.py +++ b/tests/integration/test_named_collections_if_exists_on_cluster/test.py @@ -26,9 +26,11 @@ select * from system.named_collections; """ import logging -from json import dumps, loads from functools import partial +from json import dumps, loads + import pytest + from helpers.cluster import ClickHouseCluster dumps = partial(dumps, ensure_ascii=False) diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 187cae5c957..29776eba176 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -2,6 +2,7 @@ import random import string import pytest + from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 9d4ca5ad49f..8a1a09d910f 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -1,14 +1,14 @@ +import logging import time +from multiprocessing.dummy import Pool import psycopg2 import pymysql.cursors import pytest -import logging +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -from multiprocessing.dummy import Pool cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_old_parts_finally_removed/test.py b/tests/integration/test_old_parts_finally_removed/test.py index cbd701588d5..f1acbf0e00c 100644 --- a/tests/integration/test_old_parts_finally_removed/test.py +++ b/tests/integration/test_old_parts_finally_removed/test.py @@ -3,8 +3,9 @@ import os import time -import helpers.client as client import pytest + +import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index a5e62a380bf..e1eb9257280 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_optimize_on_insert/test.py b/tests/integration/test_optimize_on_insert/test.py index 0dfec53cf9c..80dc5bbc5a9 100644 --- a/tests/integration/test_optimize_on_insert/test.py +++ b/tests/integration/test_optimize_on_insert/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_parallel_replicas_all_marks_read/test.py b/tests/integration/test_parallel_replicas_all_marks_read/test.py index e67da7fe02d..593b98126df 100644 --- a/tests/integration/test_parallel_replicas_all_marks_read/test.py +++ b/tests/integration/test_parallel_replicas_all_marks_read/test.py @@ -1,8 +1,9 @@ import json +from random import randint + import pytest from helpers.cluster import ClickHouseCluster -from random import randint cluster = ClickHouseCluster(__file__) cluster_name = "parallel_replicas_with_unavailable_nodes" diff --git a/tests/integration/test_parallel_replicas_custom_key/test.py b/tests/integration/test_parallel_replicas_custom_key/test.py index 39fd7ff292e..1e0e7724d21 100644 --- a/tests/integration/test_parallel_replicas_custom_key/test.py +++ b/tests/integration/test_parallel_replicas_custom_key/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_custom_key_failover/test.py b/tests/integration/test_parallel_replicas_custom_key_failover/test.py index d46f7955588..5c27541469b 100644 --- a/tests/integration/test_parallel_replicas_custom_key_failover/test.py +++ b/tests/integration/test_parallel_replicas_custom_key_failover/test.py @@ -1,5 +1,7 @@ -import pytest import uuid + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py b/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py index 3f462312967..8c261243508 100644 --- a/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py +++ b/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py @@ -1,5 +1,7 @@ -import pytest import uuid + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py index 0c7e820114a..3300cc15aa5 100644 --- a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py +++ b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py @@ -1,6 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_failover/test.py b/tests/integration/test_parallel_replicas_failover/test.py index b185f8a4ede..4becf656130 100644 --- a/tests/integration/test_parallel_replicas_failover/test.py +++ b/tests/integration/test_parallel_replicas_failover/test.py @@ -1,6 +1,7 @@ -import pytest import uuid +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py index 4c25315202c..6af331c795a 100644 --- a/tests/integration/test_parallel_replicas_no_replicas/test.py +++ b/tests/integration/test_parallel_replicas_no_replicas/test.py @@ -1,7 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_over_distributed/test.py b/tests/integration/test_parallel_replicas_over_distributed/test.py index 2752f1cb00f..720b8d0ad71 100644 --- a/tests/integration/test_parallel_replicas_over_distributed/test.py +++ b/tests/integration/test_parallel_replicas_over_distributed/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_protocol/test.py b/tests/integration/test_parallel_replicas_protocol/test.py index c43226c5ecc..2ed39a3273f 100644 --- a/tests/integration/test_parallel_replicas_protocol/test.py +++ b/tests/integration/test_parallel_replicas_protocol/test.py @@ -1,8 +1,9 @@ -import pytest import re +from random import randint + +import pytest from helpers.cluster import ClickHouseCluster -from random import randint cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parallel_replicas_skip_shards/test.py b/tests/integration/test_parallel_replicas_skip_shards/test.py index 0a0a1691542..d2b76c2400a 100644 --- a/tests/integration/test_parallel_replicas_skip_shards/test.py +++ b/tests/integration/test_parallel_replicas_skip_shards/test.py @@ -1,7 +1,7 @@ import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py index 778b6618a61..82584785b67 100644 --- a/tests/integration/test_parquet_page_index/test.py +++ b/tests/integration/test_parquet_page_index/test.py @@ -1,9 +1,11 @@ -import pytest -from helpers.cluster import ClickHouseCluster -import pyarrow.parquet as pq import os import time +import pyarrow.parquet as pq +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) path_to_userfiles = "/var/lib/clickhouse/user_files/" node = cluster.add_instance("node", external_dirs=[path_to_userfiles]) diff --git a/tests/integration/test_part_log_table/test.py b/tests/integration/test_part_log_table/test.py index 15d65cc31dd..2a6e12e7840 100644 --- a/tests/integration/test_part_log_table/test.py +++ b/tests/integration/test_part_log_table/test.py @@ -1,6 +1,7 @@ -import pytest import fnmatch +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 054418a8ba9..6600d6712e0 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -1,10 +1,13 @@ -import pytest import logging + +import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts +from helpers.test_tools import TSV, assert_eq_with_retry +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, +) cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_parts_delete_zookeeper/test.py b/tests/integration/test_parts_delete_zookeeper/test.py index d7b5fe1cb57..0512a744c10 100644 --- a/tests/integration/test_parts_delete_zookeeper/test.py +++ b/tests/integration/test_parts_delete_zookeeper/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py index e64ca7ece33..dd487adc47f 100644 --- a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py +++ b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_peak_memory_usage/test.py b/tests/integration/test_peak_memory_usage/test.py index a1313461482..877cf97bb18 100644 --- a/tests/integration/test_peak_memory_usage/test.py +++ b/tests/integration/test_peak_memory_usage/test.py @@ -1,6 +1,7 @@ -import pytest -import tempfile import re +import tempfile + +import pytest from helpers.cluster import ClickHouseCluster from helpers.uclient import client, prompt diff --git a/tests/integration/test_placement_info/test.py b/tests/integration/test_placement_info/test.py index 32fd2fa57d7..9a887f1532c 100644 --- a/tests/integration/test_placement_info/test.py +++ b/tests/integration/test_placement_info/test.py @@ -1,7 +1,9 @@ +import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -import os METADATA_SERVER_HOSTNAME = "node_imds" METADATA_SERVER_PORT = 8080 diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 2b30170b203..49acf287481 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -4,9 +4,9 @@ import string import struct import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION -from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry, exec_query_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index c44fa176599..1e7bd5cee2e 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -1,10 +1,10 @@ -import pytest import psycopg2 +import pytest +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry from helpers.postgres_utility import get_postgres_conn -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_postgresql_protocol/test.py b/tests/integration/test_postgresql_protocol/test.py index 9da9ec4d0b7..9396e351c63 100644 --- a/tests/integration/test_postgresql_protocol/test.py +++ b/tests/integration/test_postgresql_protocol/test.py @@ -2,13 +2,14 @@ import datetime import decimal +import logging import os import uuid -import logging import psycopg2 as py_psql import psycopg2.extras import pytest + from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check psycopg2.extras.register_uuid() diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index 0e87cb0e690..45e15b56b27 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -1,34 +1,32 @@ -import pytest - -import time import os.path as p import random +import threading +import time +from random import randrange + +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry -from helpers.test_tools import TSV - -from random import randrange -import threading - -from helpers.postgres_utility import get_postgres_conn -from helpers.postgres_utility import PostgresManager - -from helpers.postgres_utility import create_replication_slot, drop_replication_slot -from helpers.postgres_utility import create_postgres_schema, drop_postgres_schema -from helpers.postgres_utility import create_postgres_table, drop_postgres_table -from helpers.postgres_utility import check_tables_are_synchronized -from helpers.postgres_utility import check_several_tables_are_synchronized -from helpers.postgres_utility import assert_nested_table_is_created -from helpers.postgres_utility import assert_number_of_columns from helpers.postgres_utility import ( + PostgresManager, + assert_nested_table_is_created, + assert_number_of_columns, + check_several_tables_are_synchronized, + check_tables_are_synchronized, + create_postgres_schema, + create_postgres_table, + create_replication_slot, + drop_postgres_schema, + drop_postgres_table, + drop_replication_slot, + get_postgres_conn, postgres_table_template, postgres_table_template_2, postgres_table_template_3, postgres_table_template_4, + queries, ) -from helpers.postgres_utility import queries - +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 55f1beae12c..b3e16d4b150 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -1,43 +1,39 @@ -import pytest - -import uuid -import time -import psycopg2 import os.path as p import random +import threading +import time +import uuid +from random import randrange + +import psycopg2 +import pytest +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -from helpers.test_tools import TSV - -from random import randrange -import threading - -from helpers.postgres_utility import get_postgres_conn -from helpers.postgres_utility import PostgresManager - -from helpers.postgres_utility import create_replication_slot, drop_replication_slot -from helpers.postgres_utility import create_postgres_schema, drop_postgres_schema -from helpers.postgres_utility import create_postgres_table, drop_postgres_table from helpers.postgres_utility import ( + PostgresManager, + assert_nested_table_is_created, + assert_number_of_columns, + check_several_tables_are_synchronized, + check_tables_are_synchronized, + create_postgres_schema, + create_postgres_table, create_postgres_table_with_schema, + create_replication_slot, + drop_postgres_schema, + drop_postgres_table, drop_postgres_table_with_schema, -) -from helpers.postgres_utility import check_tables_are_synchronized -from helpers.postgres_utility import check_several_tables_are_synchronized -from helpers.postgres_utility import assert_nested_table_is_created -from helpers.postgres_utility import assert_number_of_columns -from helpers.postgres_utility import ( + drop_replication_slot, + get_postgres_conn, postgres_table_template, postgres_table_template_2, postgres_table_template_3, postgres_table_template_4, postgres_table_template_5, postgres_table_template_6, + queries, ) -from helpers.postgres_utility import queries - +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index a8e037d667f..827b0bb3c9c 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -3,6 +3,7 @@ import re import pytest import requests + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_profile_max_sessions_for_user/test.py b/tests/integration/test_profile_max_sessions_for_user/test.py index a2fa77e8dc9..9663cfa2e49 100755 --- a/tests/integration/test_profile_max_sessions_for_user/test.py +++ b/tests/integration/test_profile_max_sessions_for_user/test.py @@ -1,12 +1,12 @@ import os - -import grpc -import pymysql.connections -import psycopg2 as py_psql -import pytest import sys import threading +import grpc +import psycopg2 as py_psql +import pymysql.connections +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_logs_contain_with_retry from helpers.uclient import client, prompt @@ -15,8 +15,8 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) grpc_protocol_pb2_dir = os.path.join(script_dir, "grpc_protocol_pb2") if grpc_protocol_pb2_dir not in sys.path: sys.path.append(grpc_protocol_pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute grpc_protocol_pb2/generate.py to generate these modules. - +import clickhouse_grpc_pb2 # Execute grpc_protocol_pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc MAX_SESSIONS_FOR_USER = 2 POSTGRES_SERVER_PORT = 5433 diff --git a/tests/integration/test_prometheus_endpoint/test.py b/tests/integration/test_prometheus_endpoint/test.py index c1f04497b55..e702d1499d5 100644 --- a/tests/integration/test_prometheus_endpoint/test.py +++ b/tests/integration/test_prometheus_endpoint/test.py @@ -3,6 +3,7 @@ import time import pytest import requests + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_prometheus_protocols/test.py b/tests/integration/test_prometheus_protocols/test.py index 0c75a8194c7..e368c841c4e 100644 --- a/tests/integration/test_prometheus_protocols/test.py +++ b/tests/integration/test_prometheus_protocols/test.py @@ -1,7 +1,9 @@ -import pytest import time -import requests from http import HTTPStatus + +import pytest +import requests + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index b842a54741e..eefc4882e8e 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -2,6 +2,7 @@ import concurrent import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py index f30f57cc1d6..0c9a0c1a5bf 100644 --- a/tests/integration/test_quorum_inserts_parallel/test.py +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import pytest -from helpers.cluster import ClickHouseCluster from multiprocessing.dummy import Pool -from helpers.network import PartitionManager -from helpers.client import QueryRuntimeException -from helpers.test_tools import assert_eq_with_retry +import pytest + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index bb50c84b340..7cb973f9c6a 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -3,8 +3,9 @@ import re import time import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_random_inserts/test.py b/tests/integration/test_random_inserts/test.py index 9ac0c5b024c..40000c5ecd4 100644 --- a/tests/integration/test_random_inserts/test.py +++ b/tests/integration/test_random_inserts/test.py @@ -4,6 +4,7 @@ import threading import time import pytest + from helpers.client import CommandRequest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_read_only_table/test.py b/tests/integration/test_read_only_table/test.py index df084f9dbbd..e44f3b49d33 100644 --- a/tests/integration/test_read_only_table/test.py +++ b/tests/integration/test_read_only_table/test.py @@ -1,8 +1,9 @@ -import time -import re import logging +import re +import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_read_temporary_tables_on_failure/test.py b/tests/integration/test_read_temporary_tables_on_failure/test.py index 77c8f3cf26b..367a4ae969f 100644 --- a/tests/integration/test_read_temporary_tables_on_failure/test.py +++ b/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -1,5 +1,6 @@ import pytest -from helpers.client import QueryTimeoutExceedException, QueryRuntimeException + +from helpers.client import QueryRuntimeException, QueryTimeoutExceedException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_recompression_ttl/test.py b/tests/integration/test_recompression_ttl/test.py index 9d7b09eacdf..76b0b08be7b 100644 --- a/tests/integration/test_recompression_ttl/test.py +++ b/tests/integration/test_recompression_ttl/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_recovery_replica/test.py b/tests/integration/test_recovery_replica/test.py index 582e018f5d2..f68998f1c41 100644 --- a/tests/integration/test_recovery_replica/test.py +++ b/tests/integration/test_recovery_replica/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py index 6fcf2fad423..6ccd4cf738a 100644 --- a/tests/integration/test_recovery_time_metric/test.py +++ b/tests/integration/test_recovery_time_metric/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index ba3fb3e14ab..6ff51475d23 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -1,10 +1,11 @@ -import pytest -from helpers.cluster import ClickHouseCluster, is_arm - -from helpers.network import PartitionManager import threading import time +import pytest + +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.network import PartitionManager + # skip all tests in the module on ARM due to HDFS if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_reload_auxiliary_zookeepers/test.py b/tests/integration/test_reload_auxiliary_zookeepers/test.py index 476c5dee99e..5b6a1621d6b 100644 --- a/tests/integration/test_reload_auxiliary_zookeepers/test.py +++ b/tests/integration/test_reload_auxiliary_zookeepers/test.py @@ -1,9 +1,10 @@ -import time -import pytest import os +import time + +import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_reload_certificate/test.py b/tests/integration/test_reload_certificate/test.py index f0efc4e0bbd..9328cc2c008 100644 --- a/tests/integration/test_reload_certificate/test.py +++ b/tests/integration/test_reload_certificate/test.py @@ -1,5 +1,7 @@ -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_reload_max_table_size_to_drop/test.py b/tests/integration/test_reload_max_table_size_to_drop/test.py index 90e60e5cfa4..63778c876f5 100644 --- a/tests/integration/test_reload_max_table_size_to_drop/test.py +++ b/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -2,6 +2,7 @@ import os import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_reload_query_masking_rules/test.py b/tests/integration/test_reload_query_masking_rules/test.py index f269aefbacb..5ea36d19420 100644 --- a/tests/integration/test_reload_query_masking_rules/test.py +++ b/tests/integration/test_reload_query_masking_rules/test.py @@ -1,5 +1,7 @@ -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_retry diff --git a/tests/integration/test_reload_zookeeper/test.py b/tests/integration/test_reload_zookeeper/test.py index 8924376d6fd..3014c22b5f1 100644 --- a/tests/integration/test_reload_zookeeper/test.py +++ b/tests/integration/test_reload_zookeeper/test.py @@ -1,12 +1,12 @@ -import time -import pytest import os +import time + +import pytest -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") node = cluster.add_instance("node", with_zookeeper=True) diff --git a/tests/integration/test_reloading_settings_from_users_xml/test.py b/tests/integration/test_reloading_settings_from_users_xml/test.py index 3f4ba2913a2..56c1529bf10 100644 --- a/tests/integration/test_reloading_settings_from_users_xml/test.py +++ b/tests/integration/test_reloading_settings_from_users_xml/test.py @@ -1,6 +1,8 @@ -import pytest import os import time + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_retry diff --git a/tests/integration/test_reloading_storage_configuration/test.py b/tests/integration/test_reloading_storage_configuration/test.py index 4b21919ab3d..8d7c18b64fe 100644 --- a/tests/integration/test_reloading_storage_configuration/test.py +++ b/tests/integration/test_reloading_storage_configuration/test.py @@ -5,11 +5,11 @@ import shutil import time import xml.etree.ElementTree as ET +import pytest + import helpers.client import helpers.cluster from helpers.test_tools import TSV -import pytest - cluster = helpers.cluster.ClickHouseCluster(__file__) diff --git a/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py b/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py index 8c52b05dba2..d28b86eec47 100644 --- a/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py +++ b/tests/integration/test_remote_blobs_naming/test_backward_compatibility.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 +import logging +import os +import re from contextlib import contextmanager from difflib import unified_diff -import logging -import re + import pytest -import os from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_remove_stale_moving_parts/test.py b/tests/integration/test_remove_stale_moving_parts/test.py index f7cb4e5817e..e8aa6c8a8fe 100644 --- a/tests/integration/test_remove_stale_moving_parts/test.py +++ b/tests/integration/test_remove_stale_moving_parts/test.py @@ -1,6 +1,8 @@ -from pathlib import Path import time +from pathlib import Path + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 1c87b101b11..be97444660a 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -1,9 +1,10 @@ +import datetime import random import time from multiprocessing.dummy import Pool -import datetime import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_render_log_file_name_templates/test.py b/tests/integration/test_render_log_file_name_templates/test.py index 58df32b823e..a3ce5e45c8f 100644 --- a/tests/integration/test_render_log_file_name_templates/test.py +++ b/tests/integration/test_render_log_file_name_templates/test.py @@ -1,8 +1,9 @@ -import pytest import logging -from helpers.cluster import ClickHouseCluster from datetime import datetime +import pytest + +from helpers.cluster import ClickHouseCluster log_dir = "/var/log/clickhouse-server/" cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index 579b22286b9..8a1a76ee888 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -3,11 +3,11 @@ # pylint: disable=redefined-outer-name: import time + import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager - from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replica_can_become_leader/test.py b/tests/integration/test_replica_can_become_leader/test.py index 58e7b6f6e19..42ec51abb9a 100644 --- a/tests/integration/test_replica_can_become_leader/test.py +++ b/tests/integration/test_replica_can_become_leader/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_replica_is_active/test.py b/tests/integration/test_replica_is_active/test.py index d5e0931dff2..6b7f6832fc4 100644 --- a/tests/integration/test_replica_is_active/test.py +++ b/tests/integration/test_replica_is_active/test.py @@ -1,7 +1,9 @@ +from ast import literal_eval + import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from ast import literal_eval cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 533eb601ad6..20e2dbca7dd 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1,13 +1,14 @@ import os -import shutil -import time import re -import pytest +import shutil import threading +import time + +import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain test_recover_staled_replica_run = 1 diff --git a/tests/integration/test_replicated_database_alter_modify_order_by/test.py b/tests/integration/test_replicated_database_alter_modify_order_by/test.py index 22355817ee6..666d9582c16 100644 --- a/tests/integration/test_replicated_database_alter_modify_order_by/test.py +++ b/tests/integration/test_replicated_database_alter_modify_order_by/test.py @@ -1,10 +1,8 @@ import pytest - from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry - cluster = ClickHouseCluster(__file__) shard1_node = cluster.add_instance( diff --git a/tests/integration/test_replicated_database_cluster_groups/test.py b/tests/integration/test_replicated_database_cluster_groups/test.py index 797ec3d2fc7..375f5a28422 100644 --- a/tests/integration/test_replicated_database_cluster_groups/test.py +++ b/tests/integration/test_replicated_database_cluster_groups/test.py @@ -1,4 +1,5 @@ import re + import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_replicated_fetches_bandwidth/test.py b/tests/integration/test_replicated_fetches_bandwidth/test.py index 1e84c09a523..7392f70ab0c 100644 --- a/tests/integration/test_replicated_fetches_bandwidth/test.py +++ b/tests/integration/test_replicated_fetches_bandwidth/test.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -from helpers.cluster import ClickHouseCluster -import pytest import random +import statistics import string -from helpers.network import NetThroughput import subprocess import time -import statistics + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.network import NetThroughput cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True) diff --git a/tests/integration/test_replicated_fetches_timeouts/test.py b/tests/integration/test_replicated_fetches_timeouts/test.py index 55fa4b909ba..bf8b560208f 100644 --- a/tests/integration/test_replicated_fetches_timeouts/test.py +++ b/tests/integration/test_replicated_fetches_timeouts/test.py @@ -5,6 +5,7 @@ import string import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index a70f3234c1e..3c1599aeb90 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -1,5 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( diff --git a/tests/integration/test_replicated_merge_tree_config/test.py b/tests/integration/test_replicated_merge_tree_config/test.py index b5c033032ba..c9afd00c336 100644 --- a/tests/integration/test_replicated_merge_tree_config/test.py +++ b/tests/integration/test_replicated_merge_tree_config/test.py @@ -1,7 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster import logging +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", main_configs=["configs/config.xml"], with_zookeeper=True diff --git a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py index 25d30eb9c82..f36d01afd67 100644 --- a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py +++ b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py @@ -1,8 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV import os +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py index d664e77e277..edcd7507218 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py @@ -1,8 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV import os +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/test.py b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/test.py index 1c6d15d9c7b..0eb3deaad44 100644 --- a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/test.py +++ b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/test.py @@ -4,12 +4,12 @@ import logging import random import string import time - -import pytest from multiprocessing.dummy import Pool -from helpers.cluster import ClickHouseCluster -import minio +import minio +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index b90e28dfdb2..46fbf189ff1 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -3,6 +3,7 @@ import random import string import pytest + from helpers.cluster import ClickHouseCluster TABLE_NAME = "s3_test" diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 1d05eefd4c2..70f49e91d90 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -4,6 +4,7 @@ import string import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_replicated_merge_tree_thread_schedule_timeouts/test.py b/tests/integration/test_replicated_merge_tree_thread_schedule_timeouts/test.py index 515d9530424..4ed5824bf8f 100644 --- a/tests/integration/test_replicated_merge_tree_thread_schedule_timeouts/test.py +++ b/tests/integration/test_replicated_merge_tree_thread_schedule_timeouts/test.py @@ -1,8 +1,8 @@ import concurrent.futures import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster MAX_THREADS = 60 diff --git a/tests/integration/test_replicated_merge_tree_wait_on_shutdown/test.py b/tests/integration/test_replicated_merge_tree_wait_on_shutdown/test.py index 995afedf415..2aca436024f 100644 --- a/tests/integration/test_replicated_merge_tree_wait_on_shutdown/test.py +++ b/tests/integration/test_replicated_merge_tree_wait_on_shutdown/test.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 +import time +from multiprocessing.dummy import Pool + import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry -from multiprocessing.dummy import Pool -import time cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py index 5a514be58dd..9904a134ef6 100644 --- a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py @@ -1,9 +1,10 @@ import time -import helpers.client as client import pytest -from helpers.cluster import ClickHouseCluster + +import helpers.client as client from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_mutations/test.py b/tests/integration/test_replicated_mutations/test.py index e20bcf367e3..ec23502112e 100644 --- a/tests/integration/test_replicated_mutations/test.py +++ b/tests/integration/test_replicated_mutations/test.py @@ -5,6 +5,7 @@ import time from collections import Counter import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py b/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py index f16799c4667..6d2bb0a3b70 100644 --- a/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py +++ b/tests/integration/test_replicated_s3_zero_copy_drop_partition/test.py @@ -5,10 +5,10 @@ import random import string import time -import pytest -from helpers.cluster import ClickHouseCluster import minio +import pytest +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py index 4fe8064b26a..67a2ad3bca9 100644 --- a/tests/integration/test_replicated_table_attach/test.py +++ b/tests/integration/test_replicated_table_attach/test.py @@ -1,11 +1,10 @@ +import os + import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -import os - - cluster = ClickHouseCluster(__file__) CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") diff --git a/tests/integration/test_replicated_user_defined_functions/test.py b/tests/integration/test_replicated_user_defined_functions/test.py index 92d86a8fd2c..ed64bece846 100644 --- a/tests/integration/test_replicated_user_defined_functions/test.py +++ b/tests/integration/test_replicated_user_defined_functions/test.py @@ -1,13 +1,13 @@ import inspect +import os.path +import time from contextlib import nullcontext as does_not_raise import pytest -import time -import os.path -from helpers.cluster import ClickHouseCluster from helpers.client import QueryRuntimeException -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index e34495a0071..2a998b31395 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -1,10 +1,11 @@ import inspect -import pytest import time - from dataclasses import dataclass + +import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") diff --git a/tests/integration/test_replicated_zero_copy_projection_mutation/test.py b/tests/integration/test_replicated_zero_copy_projection_mutation/test.py index e0af85c3887..ae43b6fda34 100644 --- a/tests/integration/test_replicated_zero_copy_projection_mutation/test.py +++ b/tests/integration/test_replicated_zero_copy_projection_mutation/test.py @@ -1,12 +1,12 @@ import logging +import pathlib import time from contextlib import contextmanager -import pathlib import pytest -from helpers.mock_servers import start_s3_mock from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_s3_mock from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_replication_credentials/test.py b/tests/integration/test_replication_credentials/test.py index 79588fcd38b..e1ce61067d9 100644 --- a/tests/integration/test_replication_credentials/test.py +++ b/tests/integration/test_replication_credentials/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_replication_without_zookeeper/test.py b/tests/integration/test_replication_without_zookeeper/test.py index 1b2bb6ef517..cd05ec6a692 100644 --- a/tests/integration/test_replication_without_zookeeper/test.py +++ b/tests/integration/test_replication_without_zookeeper/test.py @@ -1,6 +1,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_restart_server/test.py b/tests/integration/test_restart_server/test.py index 43f3056533f..07368531adc 100755 --- a/tests/integration/test_restart_server/test.py +++ b/tests/integration/test_restart_server/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py index a975db05020..c28ff973b81 100644 --- a/tests/integration/test_restore_external_engines/test.py +++ b/tests/integration/test_restore_external_engines/test.py @@ -1,7 +1,6 @@ -import pytest - import pymysql.cursors import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_restore_replica/test.py b/tests/integration/test_restore_replica/test.py index 3d0baa87c21..0baf2817957 100644 --- a/tests/integration/test_restore_replica/test.py +++ b/tests/integration/test_restore_replica/test.py @@ -1,8 +1,8 @@ import time + import pytest -from helpers.cluster import ClickHouseCluster -from helpers.cluster import ClickHouseKiller +from helpers.cluster import ClickHouseCluster, ClickHouseKiller from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index b746af56083..0e194e37897 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -1,6 +1,8 @@ -import time -import pytest import random +import time + +import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 8260be78e82..7747d5cddd1 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -3,8 +3,9 @@ import re import time import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_runtime_configurable_cache_size/test.py b/tests/integration/test_runtime_configurable_cache_size/test.py index beaf83ea754..029753d9a64 100644 --- a/tests/integration/test_runtime_configurable_cache_size/test.py +++ b/tests/integration/test_runtime_configurable_cache_size/test.py @@ -1,6 +1,8 @@ import os import time + import pytest + from helpers.cluster import ClickHouseCluster # Tests that sizes of in-memory caches (mark / uncompressed / index mark / index uncompressed / mmapped file / query cache) can be changed diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py index 1d33ca02f86..5282d69e661 100644 --- a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from bottle import request, route, run, response +from bottle import request, response, route, run # Handle for MultipleObjectsDelete. diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py index c70d7c03a1d..ae9a6fcbe2f 100644 --- a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py @@ -4,7 +4,6 @@ import logging import os import time - import pytest from helpers.client import QueryRuntimeException diff --git a/tests/integration/test_s3_cluster/s3_mocks/s3_mock.py b/tests/integration/test_s3_cluster/s3_mocks/s3_mock.py index a8578196653..99112a65d2c 100644 --- a/tests/integration/test_s3_cluster/s3_mocks/s3_mock.py +++ b/tests/integration/test_s3_cluster/s3_mocks/s3_mock.py @@ -1,6 +1,6 @@ import sys -from bottle import route, run, request, response +from bottle import request, response, route, run @route("/<_bucket>/<_path:path>") diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index c31851fdfe9..e8bf031021e 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -1,14 +1,15 @@ -from email.errors import HeaderParseError +import csv import logging import os -import csv import shutil import time +from email.errors import HeaderParseError import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV from helpers.mock_servers import start_mock_servers +from helpers.test_tools import TSV logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) diff --git a/tests/integration/test_s3_imds/metadata_servers/server_with_session_tokens.py b/tests/integration/test_s3_imds/metadata_servers/server_with_session_tokens.py index 5dcb27d6f86..1268f5b5938 100644 --- a/tests/integration/test_s3_imds/metadata_servers/server_with_session_tokens.py +++ b/tests/integration/test_s3_imds/metadata_servers/server_with_session_tokens.py @@ -2,7 +2,6 @@ import http.server import sys import uuid - # Session tokens for IMDS sessions. tokens = set() diff --git a/tests/integration/test_s3_imds/test_session_token.py b/tests/integration/test_s3_imds/test_session_token.py index 681dc997760..07a24ba2876 100644 --- a/tests/integration/test_s3_imds/test_session_token.py +++ b/tests/integration/test_s3_imds/test_session_token.py @@ -1,7 +1,9 @@ +import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -import os METADATA_SERVER_HOSTNAME = "resolver" METADATA_SERVER_PORT = 8080 diff --git a/tests/integration/test_s3_imds/test_simple.py b/tests/integration/test_s3_imds/test_simple.py index 4884c824f99..3ea9906aaca 100644 --- a/tests/integration/test_s3_imds/test_simple.py +++ b/tests/integration/test_s3_imds/test_simple.py @@ -1,7 +1,9 @@ +import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -import os METADATA_SERVER_HOSTNAME = "resolver" METADATA_SERVER_PORT = 8080 diff --git a/tests/integration/test_s3_low_cardinality_right_border/test.py b/tests/integration/test_s3_low_cardinality_right_border/test.py index e54e6783b3d..3b06b16da6e 100644 --- a/tests/integration/test_s3_low_cardinality_right_border/test.py +++ b/tests/integration/test_s3_low_cardinality_right_border/test.py @@ -60,6 +60,7 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_s3_plain_rewritable/test.py b/tests/integration/test_s3_plain_rewritable/test.py index 020e170eb48..19f996c709c 100644 --- a/tests/integration/test_s3_plain_rewritable/test.py +++ b/tests/integration/test_s3_plain_rewritable/test.py @@ -1,8 +1,9 @@ -import pytest import random import string import threading +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_s3_storage_class/test.py b/tests/integration/test_s3_storage_class/test.py index f28ae189c92..a72db253a1b 100644 --- a/tests/integration/test_s3_storage_class/test.py +++ b/tests/integration/test_s3_storage_class/test.py @@ -1,7 +1,8 @@ -import os import logging +import os import pytest + from helpers.cluster import ClickHouseCluster logging.getLogger().setLevel(logging.INFO) diff --git a/tests/integration/test_s3_storage_conf_new_proxy/test.py b/tests/integration/test_s3_storage_conf_new_proxy/test.py index 3b3b07aaa09..68ed35bf39e 100644 --- a/tests/integration/test_s3_storage_conf_new_proxy/test.py +++ b/tests/integration/test_s3_storage_conf_new_proxy/test.py @@ -2,8 +2,9 @@ import logging import time import pytest -from helpers.cluster import ClickHouseCluster + import helpers.s3_url_proxy_tests_util as proxy_util +from helpers.cluster import ClickHouseCluster @pytest.fixture(scope="module") diff --git a/tests/integration/test_s3_storage_conf_proxy/test.py b/tests/integration/test_s3_storage_conf_proxy/test.py index da9c54066bf..0882f567c89 100644 --- a/tests/integration/test_s3_storage_conf_proxy/test.py +++ b/tests/integration/test_s3_storage_conf_proxy/test.py @@ -3,8 +3,9 @@ import os import time import pytest -from helpers.cluster import ClickHouseCluster + import helpers.s3_url_proxy_tests_util as proxy_util +from helpers.cluster import ClickHouseCluster @pytest.fixture(scope="module") diff --git a/tests/integration/test_s3_style_link/test.py b/tests/integration/test_s3_style_link/test.py index f90b77c360c..2fa7d8a9725 100644 --- a/tests/integration/test_s3_style_link/test.py +++ b/tests/integration/test_s3_style_link/test.py @@ -1,7 +1,8 @@ import logging -import pytest -from helpers.cluster import ClickHouseCluster +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_s3_table_function_with_http_proxy/test.py b/tests/integration/test_s3_table_function_with_http_proxy/test.py index 2ec73ecbef6..cb8904d36af 100644 --- a/tests/integration/test_s3_table_function_with_http_proxy/test.py +++ b/tests/integration/test_s3_table_function_with_http_proxy/test.py @@ -1,9 +1,10 @@ import logging -import helpers.s3_url_proxy_tests_util as proxy_util +import os import pytest + +import helpers.s3_url_proxy_tests_util as proxy_util from helpers.cluster import ClickHouseCluster -import os @pytest.fixture(scope="module") diff --git a/tests/integration/test_s3_table_function_with_https_proxy/test.py b/tests/integration/test_s3_table_function_with_https_proxy/test.py index 54452dda401..1588b31251f 100644 --- a/tests/integration/test_s3_table_function_with_https_proxy/test.py +++ b/tests/integration/test_s3_table_function_with_https_proxy/test.py @@ -1,8 +1,9 @@ import logging -import helpers.s3_url_proxy_tests_util as proxy_util import os import pytest + +import helpers.s3_url_proxy_tests_util as proxy_util from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_s3_table_functions/test.py b/tests/integration/test_s3_table_functions/test.py index ff62d1a9eac..b2dfddb23eb 100644 --- a/tests/integration/test_s3_table_functions/test.py +++ b/tests/integration/test_s3_table_functions/test.py @@ -1,9 +1,10 @@ import logging + import pytest + +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -from helpers.client import QueryRuntimeException - cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_s3_with_https/test.py b/tests/integration/test_s3_with_https/test.py index 6db5b7da930..8348ccb65e8 100644 --- a/tests/integration/test_s3_with_https/test.py +++ b/tests/integration/test_s3_with_https/test.py @@ -1,6 +1,7 @@ import logging import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 0ca81a27802..e723a4ffc57 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -3,6 +3,7 @@ import logging import time import pytest + from helpers.cluster import ClickHouseCluster logging.getLogger().setLevel(logging.INFO) diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py index 04bff4a44fb..ea6c1cdcb64 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test.py +++ b/tests/integration/test_s3_zero_copy_ttl/test.py @@ -2,6 +2,7 @@ import time import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py b/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py index fb9f3eb67b9..ec150707e06 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py +++ b/tests/integration/test_s3_zero_copy_ttl/test_vertical_merge_memory_usage.py @@ -2,8 +2,8 @@ import time import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster single_node_cluster = ClickHouseCluster(__file__) small_node = single_node_cluster.add_instance( diff --git a/tests/integration/test_scheduler/test.py b/tests/integration/test_scheduler/test.py index 31cc106a95d..050281b2e3a 100644 --- a/tests/integration/test_scheduler/test.py +++ b/tests/integration/test_scheduler/test.py @@ -2,8 +2,9 @@ # pylint: disable=redefined-outer-name # pylint: disable=line-too-long -import time import threading +import time + import pytest from helpers.client import QueryRuntimeException diff --git a/tests/integration/test_secure_socket/test.py b/tests/integration/test_secure_socket/test.py index 123715e5f05..d5b3cf4f184 100644 --- a/tests/integration/test_secure_socket/test.py +++ b/tests/integration/test_secure_socket/test.py @@ -2,6 +2,7 @@ import os.path import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_select_access_rights/test_from_system_tables.py b/tests/integration/test_select_access_rights/test_from_system_tables.py index 7076e5c8ce7..5fb69d26625 100644 --- a/tests/integration/test_select_access_rights/test_from_system_tables.py +++ b/tests/integration/test_select_access_rights/test_from_system_tables.py @@ -1,5 +1,7 @@ import os + import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_select_access_rights/test_main.py b/tests/integration/test_select_access_rights/test_main.py index bca3c698911..f867fb5c100 100644 --- a/tests/integration/test_select_access_rights/test_main.py +++ b/tests/integration/test_select_access_rights/test_main.py @@ -1,6 +1,7 @@ +import re + import pytest -import re from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 83c0827f891..9e17528918b 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -5,6 +5,7 @@ import os import time + import pytest import helpers.cluster diff --git a/tests/integration/test_server_keep_alive/test.py b/tests/integration/test_server_keep_alive/test.py index e550319b6df..aa3320a9cac 100644 --- a/tests/integration/test_server_keep_alive/test.py +++ b/tests/integration/test_server_keep_alive/test.py @@ -1,6 +1,7 @@ import logging -import pytest import random + +import pytest import requests from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_server_reload/test.py b/tests/integration/test_server_reload/test.py index 655c83dd81f..36c62c85d03 100644 --- a/tests/integration/test_server_reload/test.py +++ b/tests/integration/test_server_reload/test.py @@ -6,28 +6,30 @@ # pylint: disable=broad-except import contextlib +import logging +import os +import sys +import time +from pathlib import Path + import grpc import psycopg2 import pymysql.connections import pymysql.err import pytest -import sys -import os -import time -import logging -from helpers.cluster import ClickHouseCluster, run_and_check -from helpers.client import Client, QueryRuntimeException from kazoo.exceptions import NodeExistsError -from pathlib import Path from requests.exceptions import ConnectionError from urllib3.util.retry import Retry +from helpers.client import Client, QueryRuntimeException +from helpers.cluster import ClickHouseCluster, run_and_check + script_dir = os.path.dirname(os.path.realpath(__file__)) grpc_protocol_pb2_dir = os.path.join(script_dir, "grpc_protocol_pb2") if grpc_protocol_pb2_dir not in sys.path: sys.path.append(grpc_protocol_pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute grpc_protocol_pb2/generate.py to generate these modules. - +import clickhouse_grpc_pb2 # Execute grpc_protocol_pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( diff --git a/tests/integration/test_server_start_and_ip_conversions/test.py b/tests/integration/test_server_start_and_ip_conversions/test.py index abb6a546f64..a50c171377b 100644 --- a/tests/integration/test_server_start_and_ip_conversions/test.py +++ b/tests/integration/test_server_start_and_ip_conversions/test.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import logging + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_session_log/test.py b/tests/integration/test_session_log/test.py index 0eb614f7aa9..e5c11070904 100644 --- a/tests/integration/test_session_log/test.py +++ b/tests/integration/test_session_log/test.py @@ -1,20 +1,21 @@ import os -import grpc -import pymysql.connections -import pytest import random import sys import threading import time +import grpc +import pymysql.connections +import pytest + from helpers.cluster import ClickHouseCluster, run_and_check script_dir = os.path.dirname(os.path.realpath(__file__)) grpc_protocol_pb2_dir = os.path.join(script_dir, "grpc_protocol_pb2") if grpc_protocol_pb2_dir not in sys.path: sys.path.append(grpc_protocol_pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute grpc_protocol_pb2/generate.py to generate these modules. - +import clickhouse_grpc_pb2 # Execute grpc_protocol_pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc POSTGRES_SERVER_PORT = 5433 MYSQL_SERVER_PORT = 9001 diff --git a/tests/integration/test_settings_constraints/test.py b/tests/integration/test_settings_constraints/test.py index 206ae5a3aa4..f2a890a5894 100644 --- a/tests/integration/test_settings_constraints/test.py +++ b/tests/integration/test_settings_constraints/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 7bb059910dd..4a0c1943a2c 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_settings_randomization/test.py b/tests/integration/test_settings_randomization/test.py index f93a0a15984..efddc17679e 100644 --- a/tests/integration/test_settings_randomization/test.py +++ b/tests/integration/test_settings_randomization/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_sharding_key_from_default_column/test.py b/tests/integration/test_sharding_key_from_default_column/test.py index 21bccae3819..02e2def225d 100644 --- a/tests/integration/test_sharding_key_from_default_column/test.py +++ b/tests/integration/test_sharding_key_from_default_column/test.py @@ -1,5 +1,7 @@ -import pytest import itertools + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_shutdown_static_destructor_failure/test.py b/tests/integration/test_shutdown_static_destructor_failure/test.py index b1d925cc432..8933be87336 100644 --- a/tests/integration/test_shutdown_static_destructor_failure/test.py +++ b/tests/integration/test_shutdown_static_destructor_failure/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_shutdown_wait_unfinished_queries/test.py b/tests/integration/test_shutdown_wait_unfinished_queries/test.py index af86b79c387..d14c9bf632e 100644 --- a/tests/integration/test_shutdown_wait_unfinished_queries/test.py +++ b/tests/integration/test_shutdown_wait_unfinished_queries/test.py @@ -1,8 +1,9 @@ -import pytest - import threading import time import uuid + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py index 76dc254c84e..0d2f1a69424 100644 --- a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py +++ b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster, ClickHouseInstance cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_ssh_keys_authentication/test.py b/tests/integration/test_ssh_keys_authentication/test.py index da087752d29..cccdda4250c 100644 --- a/tests/integration/test_ssh_keys_authentication/test.py +++ b/tests/integration/test_ssh_keys_authentication/test.py @@ -1,5 +1,7 @@ -import pytest import os + +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 19ad67ec136..5568476cfad 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -1,13 +1,15 @@ +import logging +import os.path +import ssl +import urllib.parse +import urllib.request +from os import remove + import pytest + from helpers.client import Client from helpers.cluster import ClickHouseCluster from helpers.ssl_context import WrapSSLContextWithSNI -import urllib.request, urllib.parse -import ssl -import os.path -from os import remove -import logging - # The test cluster is configured with certificate for that host name, see 'server-ext.cnf'. # The client have to verify server certificate against that name. Client uses SNI diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 9b5c9544541..f35f303b65f 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -1,20 +1,20 @@ #!/usr/bin/env python3 import gzip +import io import json import logging import os -import io -import re import random +import re import threading import time -from azure.storage.blob import BlobServiceClient import pytest +from azure.storage.blob import BlobServiceClient + from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.test_tools import assert_logs_contain_with_retry -from helpers.test_tools import TSV +from helpers.test_tools import TSV, assert_logs_contain_with_retry @pytest.fixture(scope="module") diff --git a/tests/integration/test_storage_azure_blob_storage/test_cluster.py b/tests/integration/test_storage_azure_blob_storage/test_cluster.py index 04baf007c69..aef6e426572 100644 --- a/tests/integration/test_storage_azure_blob_storage/test_cluster.py +++ b/tests/integration/test_storage_azure_blob_storage/test_cluster.py @@ -1,22 +1,22 @@ #!/usr/bin/env python3 import gzip +import io import json import logging import os -import io import random import threading import time -from azure.storage.blob import BlobServiceClient -import helpers.client import pytest +from azure.storage.blob import BlobServiceClient + +import helpers.client from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.test_tools import TSV -from helpers.network import PartitionManager from helpers.mock_servers import start_mock_servers -from helpers.test_tools import exec_query_with_retry +from helpers.network import PartitionManager +from helpers.test_tools import TSV, exec_query_with_retry from test_storage_azure_blob_storage.test import azure_query diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index 0aa846478ea..c95193cc765 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -1,44 +1,46 @@ -import helpers.client -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV - -import pytest +import glob +import json import logging import os -import json -import time -import glob import random import string - -import pyspark -import delta -from delta import * -from pyspark.sql.types import ( - StructType, - StructField, - StringType, - IntegerType, - DateType, - TimestampType, - BooleanType, - ArrayType, -) -from pyspark.sql.functions import current_timestamp +import time from datetime import datetime -from pyspark.sql.functions import monotonically_increasing_id, row_number -from pyspark.sql.window import Window -from minio.deleteobjects import DeleteObject + +import delta import pyarrow as pa import pyarrow.parquet as pq +import pyspark +import pytest +from delta import * from deltalake.writer import write_deltalake +from minio.deleteobjects import DeleteObject +from pyspark.sql.functions import ( + current_timestamp, + monotonically_increasing_id, + row_number, +) +from pyspark.sql.types import ( + ArrayType, + BooleanType, + DateType, + IntegerType, + StringType, + StructField, + StructType, + TimestampType, +) +from pyspark.sql.window import Window +import helpers.client +from helpers.cluster import ClickHouseCluster from helpers.s3_tools import ( - prepare_s3_bucket, - upload_directory, get_file_contents, list_s3_objects, + prepare_s3_bucket, + upload_directory, ) +from helpers.test_tools import TSV SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 1bf1a4e785d..362ea7d5bda 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -1,14 +1,15 @@ import os +import re +import time +import uuid import pytest -import uuid -import time -import re -from helpers.cluster import ClickHouseCluster, is_arm -from helpers.client import QueryRuntimeException -from helpers.test_tools import TSV from pyhdfs import HdfsClient +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.test_tools import TSV + if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_storage_hudi/test.py b/tests/integration/test_storage_hudi/test.py index 0c3fbfb3cda..37dca9b3c6f 100644 --- a/tests/integration/test_storage_hudi/test.py +++ b/tests/integration/test_storage_hudi/test.py @@ -1,29 +1,31 @@ -import logging -import pytest -import os import json +import logging +import os +from datetime import datetime + +import pyspark +import pytest +from pyspark.sql.functions import ( + current_timestamp, + monotonically_increasing_id, + row_number, +) +from pyspark.sql.types import ( + ArrayType, + BooleanType, + DateType, + IntegerType, + StringType, + StructField, + StructType, + TimestampType, +) +from pyspark.sql.window import Window import helpers.client from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.s3_tools import get_file_contents, prepare_s3_bucket, upload_directory from helpers.test_tools import TSV -from helpers.s3_tools import prepare_s3_bucket, upload_directory, get_file_contents - -import pyspark -from pyspark.sql.types import ( - StructType, - StructField, - StringType, - IntegerType, - DateType, - TimestampType, - BooleanType, - ArrayType, -) -from pyspark.sql.functions import current_timestamp -from datetime import datetime -from pyspark.sql.functions import monotonically_increasing_id, row_number -from pyspark.sql.window import Window - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index 176c7e209bd..b2306d487ec 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -1,43 +1,44 @@ -import helpers.client -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.test_tools import TSV - -import pyspark +import glob +import json import logging import os -import json -import pytest import time -import glob import uuid -import os - -from pyspark.sql.types import ( - StructType, - StructField, - StringType, - IntegerType, - DateType, - TimestampType, - BooleanType, - ArrayType, -) -from pyspark.sql.functions import current_timestamp from datetime import datetime -from pyspark.sql.functions import monotonically_increasing_id, row_number -from pyspark.sql.window import Window -from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2 -from minio.deleteobjects import DeleteObject -from azure.storage.blob import BlobServiceClient +import pyspark +import pytest +from azure.storage.blob import BlobServiceClient +from minio.deleteobjects import DeleteObject +from pyspark.sql.functions import ( + current_timestamp, + monotonically_increasing_id, + row_number, +) +from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2 +from pyspark.sql.types import ( + ArrayType, + BooleanType, + DateType, + IntegerType, + StringType, + StructField, + StructType, + TimestampType, +) +from pyspark.sql.window import Window + +import helpers.client +from helpers.cluster import ClickHouseCluster, ClickHouseInstance from helpers.s3_tools import ( - prepare_s3_bucket, - get_file_contents, - list_s3_objects, - S3Uploader, AzureUploader, LocalUploader, + S3Uploader, + get_file_contents, + list_s3_objects, + prepare_s3_bucket, ) +from helpers.test_tools import TSV SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index bef90e1b9d3..814b17e1e73 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1,45 +1,43 @@ +import ast +import io import json +import logging +import math import os.path as p import random import socket +import string import threading import time -import logging -import io -import string -import ast -import math +from contextlib import contextmanager -import avro.schema -import avro.io import avro.datafile +import avro.io +import avro.schema +import kafka.errors +import pytest from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, ) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer - -import kafka.errors -import pytest from google.protobuf.internal.encoder import _VarintBytes +from kafka import BrokerConnection, KafkaAdminClient, KafkaConsumer, KafkaProducer +from kafka.admin import NewTopic +from kafka.protocol.admin import DescribeGroupsRequest_v1 +from kafka.protocol.group import MemberAssignment + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager from helpers.test_tools import TSV -from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection -from kafka.protocol.admin import DescribeGroupsRequest_v1 -from kafka.protocol.group import MemberAssignment -from kafka.admin import NewTopic -from contextlib import contextmanager +from . import kafka_pb2, message_with_repeated_pb2, social_pb2 # protoc --version # libprotoc 3.0.0 # # to create kafka_pb2.py # protoc --python_out=. kafka.proto -from . import kafka_pb2 -from . import social_pb2 -from . import message_with_repeated_pb2 if is_arm(): pytestmark = pytest.mark.skip @@ -1890,7 +1888,15 @@ def test_kafka_recreate_kafka_table(kafka_cluster, create_query_generator, log_l ) # data was not flushed yet (it will be flushed 7.5 sec after creating MV) - assert int(instance.query("SELECT count() FROM test.view")) == 240 + assert ( + int( + instance.query_with_retry( + sql="SELECT count() FROM test.view", + check_callback=lambda x: x == 240, + ) + ) + == 240 + ) instance.query( """ diff --git a/tests/integration/test_storage_kafka/test_produce_http_interface.py b/tests/integration/test_storage_kafka/test_produce_http_interface.py index fc10a07f239..0afa7dae00f 100644 --- a/tests/integration/test_storage_kafka/test_produce_http_interface.py +++ b/tests/integration/test_storage_kafka/test_produce_http_interface.py @@ -1,12 +1,13 @@ -import time import logging +import time import pytest -from helpers.cluster import ClickHouseCluster, is_arm -from helpers.test_tools import TSV from kafka import KafkaAdminClient from kafka.admin import NewTopic +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.test_tools import TSV + if is_arm(): pytestmark = pytest.mark.skip diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index a00914543c6..671308b10f2 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -1,23 +1,23 @@ +import json +import logging import os.path as p import random +import socket +import subprocess import threading import time -import pytest -import logging -from helpers.cluster import ClickHouseCluster, is_arm -from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException -from helpers.network import PartitionManager - -import json -import subprocess import kafka.errors -from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection +import pytest +from kafka import BrokerConnection, KafkaAdminClient, KafkaConsumer, KafkaProducer from kafka.admin import NewTopic -from kafka.protocol.admin import DescribeGroupsResponse_v1, DescribeGroupsRequest_v1 +from kafka.protocol.admin import DescribeGroupsRequest_v1, DescribeGroupsResponse_v1 from kafka.protocol.group import MemberAssignment -import socket + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, is_arm +from helpers.network import PartitionManager +from helpers.test_tools import TSV if is_arm(): # skip due to no arm support for clickhouse/kerberos-kdc docker image diff --git a/tests/integration/test_storage_mongodb_legacy/test.py b/tests/integration/test_storage_mongodb_legacy/test.py index c6e1c22379d..ddfd21279c9 100644 --- a/tests/integration/test_storage_mongodb_legacy/test.py +++ b/tests/integration/test_storage_mongodb_legacy/test.py @@ -1,11 +1,11 @@ -import pymongo +import datetime from uuid import UUID +import pymongo import pytest -from helpers.client import QueryRuntimeException +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -import datetime @pytest.fixture(scope="module") diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index c724c5bb498..2fc62d7f511 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -1,12 +1,13 @@ +import threading +import time from contextlib import contextmanager ## sudo -H pip install PyMySQL import pymysql.cursors import pytest -import time -import threading -from helpers.cluster import ClickHouseCluster + from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_storage_numbers/test.py b/tests/integration/test_storage_numbers/test.py index cbd7793fd8c..1836f2273d7 100644 --- a/tests/integration/test_storage_numbers/test.py +++ b/tests/integration/test_storage_numbers/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_storage_policies/test.py b/tests/integration/test_storage_policies/test.py index 389146b2171..3a16631e75f 100644 --- a/tests/integration/test_storage_policies/test.py +++ b/tests/integration/test_storage_policies/test.py @@ -1,8 +1,9 @@ import os import pytest -from helpers.test_tools import TSV + from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) CONFIG_DIR = os.path.join(SCRIPT_DIR, "configs") diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 12823f1f72d..5e1814a4ae0 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -1,7 +1,8 @@ import logging -import pytest from multiprocessing.dummy import Pool +import pytest + from helpers.cluster import ClickHouseCluster from helpers.postgres_utility import get_postgres_conn diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index bc255b1964e..253c140340d 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1,17 +1,17 @@ -import pytest - import json +import logging +import math import os.path as p import random import subprocess import threading -import logging import time from random import randrange -import math import pika +import pytest from google.protobuf.internal.encoder import _VarintBytes + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster, check_rabbitmq_is_available from helpers.test_tools import TSV diff --git a/tests/integration/test_storage_redis/test.py b/tests/integration/test_storage_redis/test.py index 2fd97b9bebd..ea0129da3ec 100644 --- a/tests/integration/test_storage_redis/test.py +++ b/tests/integration/test_storage_redis/test.py @@ -1,9 +1,10 @@ ## sudo -H pip install redis -import redis -import pytest import struct import sys +import pytest +import redis + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_storage_s3/s3_mocks/mock_s3.py b/tests/integration/test_storage_s3/s3_mocks/mock_s3.py index b31827e32bc..728a86b6921 100644 --- a/tests/integration/test_storage_s3/s3_mocks/mock_s3.py +++ b/tests/integration/test_storage_s3/s3_mocks/mock_s3.py @@ -1,6 +1,6 @@ import sys -from bottle import route, run, request, response +from bottle import request, response, route, run @route("/redirected/<_path:path>") diff --git a/tests/integration/test_storage_s3/s3_mocks/no_list_objects.py b/tests/integration/test_storage_s3/s3_mocks/no_list_objects.py index 1f6d0435872..eec817e0eb3 100644 --- a/tests/integration/test_storage_s3/s3_mocks/no_list_objects.py +++ b/tests/integration/test_storage_s3/s3_mocks/no_list_objects.py @@ -5,7 +5,6 @@ import socketserver import sys import urllib.parse - UPSTREAM_HOST = "minio1:9001" random.seed("No list objects/1.0") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 7209a9ba3ab..c721e2ff7b3 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1,19 +1,20 @@ import gzip -import uuid +import io import logging import os -import io import random import threading import time +import uuid + +import pytest import helpers.client -import pytest from helpers.cluster import ClickHouseCluster, ClickHouseInstance -from helpers.network import PartitionManager from helpers.mock_servers import start_mock_servers -from helpers.test_tools import exec_query_with_retry +from helpers.network import PartitionManager from helpers.s3_tools import prepare_s3_bucket +from helpers.test_tools import exec_query_with_retry MINIO_INTERNAL_PORT = 9001 diff --git a/tests/integration/test_storage_s3/test_invalid_env_credentials.py b/tests/integration/test_storage_s3/test_invalid_env_credentials.py index d91cb7d68f9..d0c577f5105 100644 --- a/tests/integration/test_storage_s3/test_invalid_env_credentials.py +++ b/tests/integration/test_storage_s3/test_invalid_env_credentials.py @@ -1,11 +1,12 @@ import json -import os import logging +import os + +import pytest import helpers.client -from helpers.mock_servers import start_mock_servers -import pytest from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.mock_servers import start_mock_servers MINIO_INTERNAL_PORT = 9001 diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 7b7149a4744..5e93c991aed 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -1,15 +1,15 @@ import io +import json import logging import random import string import time - -import pytest -from helpers.client import QueryRuntimeException -from helpers.cluster import ClickHouseCluster, ClickHouseInstance -import json from uuid import uuid4 +import pytest + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, ClickHouseInstance AVAILABLE_MODES = ["unordered", "ordered"] DEFAULT_AUTH = ["'minio'", "'minio123'"] diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py index 7ff7a871413..1cc2e283e07 100644 --- a/tests/integration/test_storage_url/test.py +++ b/tests/integration/test_storage_url/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py index 3c62112a7d3..04ca792aa2d 100644 --- a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py +++ b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py @@ -1,6 +1,6 @@ import http.server -import sys import json +import sys RESULT_PATH = "/echo_server_headers.txt" diff --git a/tests/integration/test_storage_url_http_headers/test.py b/tests/integration/test_storage_url_http_headers/test.py index 8fc08ec5c9d..56585298b83 100644 --- a/tests/integration/test_storage_url_http_headers/test.py +++ b/tests/integration/test_storage_url_http_headers/test.py @@ -1,10 +1,11 @@ -import pytest import os -from . import http_headers_echo_server -from . import redirect_server + +import pytest from helpers.cluster import ClickHouseCluster +from . import http_headers_echo_server, redirect_server + cluster = ClickHouseCluster(__file__) server = cluster.add_instance("node") diff --git a/tests/integration/test_storage_url_with_proxy/test.py b/tests/integration/test_storage_url_with_proxy/test.py index 107aa426836..9ca8d4309b7 100644 --- a/tests/integration/test_storage_url_with_proxy/test.py +++ b/tests/integration/test_storage_url_with_proxy/test.py @@ -1,11 +1,12 @@ +import base64 +import hashlib +import hmac import logging import time from datetime import datetime -import hmac -import hashlib -import base64 import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_structured_logging_json/test.py b/tests/integration/test_structured_logging_json/test.py index 6d1455f6e0e..a7d048cc4bb 100644 --- a/tests/integration/test_structured_logging_json/test.py +++ b/tests/integration/test_structured_logging_json/test.py @@ -1,8 +1,10 @@ -import pytest -from helpers.cluster import ClickHouseCluster import json from xml.etree import ElementTree as ET +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node_all_keys = cluster.add_instance( "node_all_keys", main_configs=["configs/config_all_keys_json.xml"] diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py index 2eb870efcbc..6473eeb338f 100644 --- a/tests/integration/test_system_detached_tables/test.py +++ b/tests/integration/test_system_detached_tables/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py index cfecea5b3d6..8519a1e94e9 100644 --- a/tests/integration/test_system_flush_logs/test.py +++ b/tests/integration/test_system_flush_logs/test.py @@ -3,8 +3,9 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_retry, TSV +from helpers.test_tools import TSV, assert_eq_with_retry, assert_logs_contain_with_retry cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_system_logs/test_system_logs.py b/tests/integration/test_system_logs/test_system_logs.py index 1c45e69957b..f3976b51480 100644 --- a/tests/integration/test_system_logs/test_system_logs.py +++ b/tests/integration/test_system_logs/test_system_logs.py @@ -3,6 +3,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_system_logs_comment/test.py b/tests/integration/test_system_logs_comment/test.py index 0659a2689a0..ade9b798b62 100644 --- a/tests/integration/test_system_logs_comment/test.py +++ b/tests/integration/test_system_logs_comment/test.py @@ -3,6 +3,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_system_logs_hostname/test_replicated.py b/tests/integration/test_system_logs_hostname/test_replicated.py index b9db6b03673..54afd970f0f 100644 --- a/tests/integration/test_system_logs_hostname/test_replicated.py +++ b/tests/integration/test_system_logs_hostname/test_replicated.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_system_logs_recreate/test.py b/tests/integration/test_system_logs_recreate/test.py index 8b84734ed02..cb68028b42e 100644 --- a/tests/integration/test_system_logs_recreate/test.py +++ b/tests/integration/test_system_logs_recreate/test.py @@ -3,6 +3,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 6dbe6c891f2..b68803bf1b6 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -2,6 +2,7 @@ import threading import time import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_system_metrics/test.py b/tests/integration/test_system_metrics/test.py index e59ed919708..73b8ac4b10a 100644 --- a/tests/integration/test_system_metrics/test.py +++ b/tests/integration/test_system_metrics/test.py @@ -1,12 +1,12 @@ import time import pytest -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry -from helpers.network import PartitionManager - from kazoo.client import KazooClient +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry + def fill_nodes(nodes, shard): for node in nodes: diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index 9138a934554..7c32986b339 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -6,9 +6,9 @@ from contextlib import contextmanager import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException @pytest.fixture(scope="module") diff --git a/tests/integration/test_system_reload_async_metrics/test.py b/tests/integration/test_system_reload_async_metrics/test.py index f0572cd2db6..4305780a7d2 100644 --- a/tests/integration/test_system_reload_async_metrics/test.py +++ b/tests/integration/test_system_reload_async_metrics/test.py @@ -1,7 +1,9 @@ import os -import pytest import shutil import time + +import pytest + from helpers.cluster import ClickHouseCluster # Tests that SYSTEM RELOAD ASYNCHRONOUS METRICS works. diff --git a/tests/integration/test_system_replicated_fetches/test.py b/tests/integration/test_system_replicated_fetches/test.py index ab6808dca89..8cb571c3c58 100644 --- a/tests/integration/test_system_replicated_fetches/test.py +++ b/tests/integration/test_system_replicated_fetches/test.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import pytest +import json +import random +import string import time + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry -import random -import string -import json cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True) diff --git a/tests/integration/test_system_start_stop_listen/test.py b/tests/integration/test_system_start_stop_listen/test.py index 391cb757519..dbd306444d7 100644 --- a/tests/integration/test_system_start_stop_listen/test.py +++ b/tests/integration/test_system_start_stop_listen/test.py @@ -2,11 +2,13 @@ import os + import pytest -from helpers.cluster import ClickHouseCluster -from helpers.client import Client, QueryRuntimeException import requests +from helpers.client import Client, QueryRuntimeException +from helpers.cluster import ClickHouseCluster + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py index b7f9d7c0b96..b3aff6ddca2 100644 --- a/tests/integration/test_table_db_num_limit/test.py +++ b/tests/integration/test_table_db_num_limit/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_table_function_mongodb/test.py b/tests/integration/test_table_function_mongodb/test.py index 449e9d90672..b3e990cf73b 100644 --- a/tests/integration/test_table_function_mongodb/test.py +++ b/tests/integration/test_table_function_mongodb/test.py @@ -1,8 +1,7 @@ import pymongo - import pytest -from helpers.client import QueryRuntimeException +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_table_function_mongodb_legacy/test.py b/tests/integration/test_table_function_mongodb_legacy/test.py index a3dcf84193e..353ef4d1dff 100644 --- a/tests/integration/test_table_function_mongodb_legacy/test.py +++ b/tests/integration/test_table_function_mongodb_legacy/test.py @@ -1,8 +1,7 @@ import pymongo - import pytest -from helpers.client import QueryRuntimeException +from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_table_function_redis/test.py b/tests/integration/test_table_function_redis/test.py index f4bcebe2f90..236dfac9067 100644 --- a/tests/integration/test_table_function_redis/test.py +++ b/tests/integration/test_table_function_redis/test.py @@ -1,9 +1,9 @@ import datetime - -import redis -import pytest -import sys import struct +import sys + +import pytest +import redis from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_table_functions_access_rights/test.py b/tests/integration/test_table_functions_access_rights/test.py index b1d1a291bc5..7bc02ab902e 100644 --- a/tests/integration/test_table_functions_access_rights/test.py +++ b/tests/integration/test_table_functions_access_rights/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_tcp_handler_http_responses/test_case.py b/tests/integration/test_tcp_handler_http_responses/test_case.py index 98f4b74223e..4fadf085f38 100644 --- a/tests/integration/test_tcp_handler_http_responses/test_case.py +++ b/tests/integration/test_tcp_handler_http_responses/test_case.py @@ -1,10 +1,12 @@ """Test HTTP responses given by the TCP Handler.""" from pathlib import Path + import pytest -from helpers.cluster import ClickHouseCluster import requests +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node_with_http = cluster.add_instance( diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py index b20ab48795b..91086502381 100644 --- a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py +++ b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py @@ -1,11 +1,13 @@ """Test Interserver responses on configured IP.""" -from pathlib import Path -import pytest -from helpers.cluster import ClickHouseCluster -import requests import socket import time +from pathlib import Path + +import pytest +import requests + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_temporary_data/test.py b/tests/integration/test_temporary_data/test.py index 9228da0698f..0f47a3d7b00 100644 --- a/tests/integration/test_temporary_data/test.py +++ b/tests/integration/test_temporary_data/test.py @@ -1,9 +1,10 @@ # pylint: disable=unused-argument # pylint: disable=redefined-outer-name -import pytest import time +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py index 87192a20975..3be0f971f67 100644 --- a/tests/integration/test_temporary_data_in_cache/test.py +++ b/tests/integration/test_temporary_data_in_cache/test.py @@ -1,12 +1,12 @@ # pylint: disable=unused-argument # pylint: disable=redefined-outer-name -import pytest import fnmatch -from helpers.cluster import ClickHouseCluster -from helpers.client import QueryRuntimeException +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster MB = 1024 * 1024 diff --git a/tests/integration/test_text_log_level/test.py b/tests/integration/test_text_log_level/test.py index dc0ae6333d6..1af89e7017c 100644 --- a/tests/integration/test_text_log_level/test.py +++ b/tests/integration/test_text_log_level/test.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_throttling/test.py b/tests/integration/test_throttling/test.py index 4bd96e2756d..a2b05291463 100644 --- a/tests/integration/test_throttling/test.py +++ b/tests/integration/test_throttling/test.py @@ -16,6 +16,7 @@ # - and that max_backup_bandwidth from the query will override setting from the user profile import time + import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index f88ff5da814..5bdd316a7b7 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -1,11 +1,13 @@ +import logging +import os.path +import ssl +import urllib.parse +import urllib.request + import pytest + from helpers.cluster import ClickHouseCluster from helpers.ssl_context import WrapSSLContextWithSNI -import urllib.request, urllib.parse -import ssl -import os.path -import logging - # The test cluster is configured with certificate for that host name, see 'server-ext.cnf'. # The client has to verify server certificate against that name. Client uses SNI diff --git a/tests/integration/test_trace_collector_serverwide/test.py b/tests/integration/test_trace_collector_serverwide/test.py index 9bd107ac365..838b42912dd 100644 --- a/tests/integration/test_trace_collector_serverwide/test.py +++ b/tests/integration/test_trace_collector_serverwide/test.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import pytest import time +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_trace_log_build_id/test.py b/tests/integration/test_trace_log_build_id/test.py index 8d654aef342..e74aa7d49f9 100644 --- a/tests/integration/test_trace_log_build_id/test.py +++ b/tests/integration/test_trace_log_build_id/test.py @@ -1,5 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster TEST_QUERY_ID = "test_trace_log_build_id_query_{}" OLD_TEST_QUERY_ID = TEST_QUERY_ID.format("0") diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py index d63b7b6f545..6db04f379f2 100644 --- a/tests/integration/test_transactions/test.py +++ b/tests/integration/test_transactions/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_truncate_database/test_distributed.py b/tests/integration/test_truncate_database/test_distributed.py index 6b822077a3c..62678027e10 100644 --- a/tests/integration/test_truncate_database/test_distributed.py +++ b/tests/integration/test_truncate_database/test_distributed.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_truncate_database/test_replicated.py b/tests/integration/test_truncate_database/test_replicated.py index edcc0446da3..73be1461a61 100644 --- a/tests/integration/test_truncate_database/test_replicated.py +++ b/tests/integration/test_truncate_database/test_replicated.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 925bdf9baaa..67fe57310fa 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -3,13 +3,13 @@ import random import threading import time from multiprocessing.dummy import Pool -from helpers.test_tools import assert_logs_contain_with_retry import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_retry # FIXME: each sleep(1) is a time bomb, and not only this cause false positive # it also makes the test not reliable (i.e. assertions may be wrong, due timing issues) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 538322473ee..939453178fc 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -1,12 +1,14 @@ import time -import helpers.client as client import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION -from helpers.test_tools import TSV, exec_query_with_retry -from helpers.wait_for_helpers import wait_for_delete_inactive_parts -from helpers.wait_for_helpers import wait_for_delete_empty_parts -from helpers.test_tools import assert_eq_with_retry + +import helpers.client as client +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry, exec_query_with_retry +from helpers.wait_for_helpers import ( + wait_for_delete_empty_parts, + wait_for_delete_inactive_parts, +) cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", with_zookeeper=True) diff --git a/tests/integration/test_unambiguous_alter_commands/test.py b/tests/integration/test_unambiguous_alter_commands/test.py index 768ab78fbd8..5bf9e32c589 100644 --- a/tests/integration/test_unambiguous_alter_commands/test.py +++ b/tests/integration/test_unambiguous_alter_commands/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_undrop_query/test.py b/tests/integration/test_undrop_query/test.py index 1e77fc8db04..b432a5d0b2a 100644 --- a/tests/integration/test_undrop_query/test.py +++ b/tests/integration/test_undrop_query/test.py @@ -1,7 +1,8 @@ -import pytest -import uuid import logging import time +import uuid + +import pytest from helpers.cluster import ClickHouseCluster diff --git a/tests/integration/test_unknown_column_dist_table_with_alias/test.py b/tests/integration/test_unknown_column_dist_table_with_alias/test.py index 0d3890f3e09..f3bb30afe99 100644 --- a/tests/integration/test_unknown_column_dist_table_with_alias/test.py +++ b/tests/integration/test_unknown_column_dist_table_with_alias/test.py @@ -1,7 +1,9 @@ -import pytest -from helpers.cluster import ClickHouseCluster import logging +import pytest + +from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", main_configs=["configs/clusters.xml"]) diff --git a/tests/integration/test_user_defined_object_persistence/test.py b/tests/integration/test_user_defined_object_persistence/test.py index eb6b3621a54..e21c8d52706 100644 --- a/tests/integration/test_user_defined_object_persistence/test.py +++ b/tests/integration/test_user_defined_object_persistence/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_user_directories/test.py b/tests/integration/test_user_directories/test.py index 704fb30b2fd..0fa5a7e3dc5 100644 --- a/tests/integration/test_user_directories/test.py +++ b/tests/integration/test_user_directories/test.py @@ -1,6 +1,7 @@ import os import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_user_grants_from_config/test.py b/tests/integration/test_user_grants_from_config/test.py index d2bd0b0facd..bd226f4850d 100644 --- a/tests/integration/test_user_grants_from_config/test.py +++ b/tests/integration/test_user_grants_from_config/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index 50b7dd098c9..eea05af9e45 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -1,7 +1,8 @@ -import pytest from datetime import datetime, timedelta from time import sleep +import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index b386a79c932..fd1bfb2ba84 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -1,6 +1,6 @@ import pytest -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 9365498f89d..9aea0c0844b 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -1,7 +1,8 @@ -import pytest import time -from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION +import pytest + +from helpers.cluster import CLICKHOUSE_CI_MIN_TESTED_VERSION, ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_wrong_db_or_table_name/test.py b/tests/integration/test_wrong_db_or_table_name/test.py index 4a6dcf5aa41..93c9b512bb8 100644 --- a/tests/integration/test_wrong_db_or_table_name/test.py +++ b/tests/integration/test_wrong_db_or_table_name/test.py @@ -1,4 +1,5 @@ import pytest + from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py index dc79e5d8723..eabd0b2d50c 100644 --- a/tests/integration/test_zero_copy_fetch/test.py +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -4,11 +4,11 @@ import logging import random import string import time - from multiprocessing.dummy import Pool -import pytest -from helpers.cluster import ClickHouseCluster +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_zero_copy_replication_drop_detached_part/test.py b/tests/integration/test_zero_copy_replication_drop_detached_part/test.py index 2ecf026d9fb..29c1e9c4389 100644 --- a/tests/integration/test_zero_copy_replication_drop_detached_part/test.py +++ b/tests/integration/test_zero_copy_replication_drop_detached_part/test.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 import logging +import os import random import string import time -import os - from multiprocessing.dummy import Pool -import pytest -from helpers.cluster import ClickHouseCluster +import pytest + +from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_zero_copy_unfreeze/test.py b/tests/integration/test_zero_copy_unfreeze/test.py index 9c1902c97ee..68a7c708131 100644 --- a/tests/integration/test_zero_copy_unfreeze/test.py +++ b/tests/integration/test_zero_copy_unfreeze/test.py @@ -1,4 +1,5 @@ from collections.abc import Iterable + import pytest from helpers.cluster import ClickHouseCluster, ClickHouseInstance diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 0c0f77ec597..96d614ad446 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -1,6 +1,8 @@ -import time -import pytest import logging +import time + +import pytest + from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index 55a06cd5f51..4af14974f61 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -1,5 +1,7 @@ import time + import pytest + from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_zookeeper_config/test_secure.py b/tests/integration/test_zookeeper_config/test_secure.py index 1bc7c62f92c..80883e2fd9f 100644 --- a/tests/integration/test_zookeeper_config/test_secure.py +++ b/tests/integration/test_zookeeper_config/test_secure.py @@ -1,8 +1,9 @@ -import threading import os +import threading from tempfile import NamedTemporaryFile import pytest + from helpers.cluster import ClickHouseCluster TEST_DIR = os.path.dirname(__file__) diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py index cc0a9022674..0fa4a90d51a 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/test.py +++ b/tests/integration/test_zookeeper_config_load_balancing/test.py @@ -1,5 +1,7 @@ import time + import pytest + from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry diff --git a/tests/integration/test_zookeeper_fallback_session/test.py b/tests/integration/test_zookeeper_fallback_session/test.py index 932bbe482d2..99e1b2938f4 100644 --- a/tests/integration/test_zookeeper_fallback_session/test.py +++ b/tests/integration/test_zookeeper_fallback_session/test.py @@ -1,8 +1,8 @@ import pytest + from helpers.cluster import ClickHouseCluster, ClickHouseInstance from helpers.network import PartitionManager - cluster = ClickHouseCluster( __file__, zookeeper_config_path="configs/zookeeper_load_balancing.xml" ) diff --git a/tests/jepsen.clickhouse/resources/keeper_config.xml b/tests/jepsen.clickhouse/resources/keeper_config.xml index b5c0aac6a1b..aca5fbb87d4 100644 --- a/tests/jepsen.clickhouse/resources/keeper_config.xml +++ b/tests/jepsen.clickhouse/resources/keeper_config.xml @@ -34,6 +34,7 @@ 9181 {id} 1 + 1 10000 diff --git a/tests/performance/insert_sparse_column.xml b/tests/performance/insert_sparse_column.xml new file mode 100644 index 00000000000..0f6cdcec332 --- /dev/null +++ b/tests/performance/insert_sparse_column.xml @@ -0,0 +1,17 @@ + + CREATE TABLE t_insert_sparse (id UInt64, c0 String, c1 String, c2 String, c3 String, c4 String, c5 String, c6 String, c7 String, c8 String, c9 String, c10 String, c11 String, c12 String, c13 String, c14 String, c15 String, c16 String, c17 String, c18 String, c19 String, c20 String, c21 String, c22 String, c23 String, c24 String, c25 String, c26 String, c27 String, c28 String, c29 String, c30 String, c31 String, c32 String, c33 String, c34 String, c35 String, c36 String, c37 String, c38 String, c39 String, c40 String, c41 String, c42 String, c43 String, c44 String, c45 String, c46 String, c47 String, c48 String, c49 String, c50 String, c51 String, c52 String, c53 String, c54 String, c55 String, c56 String, c57 String, c58 String, c59 String, c60 String, c61 String, c62 String, c63 String, c64 String, c65 String, c66 String, c67 String, c68 String, c69 String, c70 String, c71 String, c72 String, c73 String, c74 String, c75 String, c76 String, c77 String, c78 String, c79 String, c80 String, c81 String, c82 String, c83 String, c84 String, c85 String, c86 String, c87 String, c88 String, c89 String, c90 String, c91 String, c92 String, c93 String, c94 String, c95 String, c96 String, c97 String, c98 String, c99 String) ENGINE = MergeTree ORDER BY id + + SYSTEM STOP MERGES t_insert_sparse + + + INSERT INTO FUNCTION file('test_data_sparse.json', LineAsString) + SELECT '{{"id": ' || number || ', "c' || number % 50 || '": "' || hex(rand()) || '"}}' + FROM numbers(100000) SETTINGS engine_file_truncate_on_insert = 1 + + + INSERT INTO t_insert_sparse SELECT * FROM file('test_data_sparse.json', JSONEachRow) + + INSERT INTO t_insert_sparse SELECT * FROM file('test_data_sparse.json', JSONEachRow) + + DROP TABLE IF EXISTS t_insert_sparse + diff --git a/tests/performance/point_in_polygon_index.xml b/tests/performance/point_in_polygon_index.xml new file mode 100644 index 00000000000..8706d643345 --- /dev/null +++ b/tests/performance/point_in_polygon_index.xml @@ -0,0 +1,21 @@ + + + + 0 + + + CREATE TABLE polygons(`x` Float64, `y` Float64, INDEX mm_x_y (x, y) TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY x + + INSERT INTO polygons + SELECT toFloat64(1000000) / ((number % 100000) + 1), toFloat64(1000000) / ((number % 100000) + 1) from system.numbers LIMIT 10000000 + SETTINGS max_insert_threads = 2, max_memory_usage = 30000000000; + + + SELECT count(*) FROM polygons + WHERE pointInPolygon((x, y), [(1,0),(0.9993908270190958,0.03489949670250097),(0.9975640502598242,0.0697564737441253),(0.9945218953682733,0.10452846326765346),(0.9902680687415704,0.13917310096006544),(0.984807753012208,0.17364817766693033),(0.9781476007338057,0.20791169081775931),(0.9702957262759965,0.24192189559966773),(0.9612616959383189,0.27563735581699916),(0.9510565162951535,0.3090169943749474),(0.9396926207859084,0.3420201433256687),(0.9271838545667874,0.374606593415912),(0.9135454576426009,0.40673664307580015),(0.898794046299167,0.4383711467890774),(0.882947592858927,0.4694715627858908),(0.8660254037844387,0.49999999999999994),(0.848048096156426,0.5299192642332049),(0.8290375725550417,0.5591929034707468),(0.8090169943749475,0.5877852522924731),(0.7880107536067219,0.6156614753256583),(0.766044443118978,0.6427876096865393),(0.7431448254773942,0.6691306063588582),(0.7193398003386512,0.6946583704589973),(0.6946583704589974,0.7193398003386511),(0.6691306063588582,0.7431448254773941),(0.6427876096865394,0.766044443118978),(0.6156614753256583,0.7880107536067219),(0.5877852522924731,0.8090169943749475),(0.5591929034707468,0.8290375725550417),(0.5299192642332049,0.848048096156426),(0.5000000000000001,0.8660254037844386),(0.46947156278589086,0.8829475928589269),(0.43837114678907746,0.898794046299167),(0.4067366430758004,0.9135454576426009),(0.3746065934159122,0.9271838545667873),(0.3420201433256688,0.9396926207859083),(0.30901699437494745,0.9510565162951535),(0.27563735581699916,0.9612616959383189),(0.24192189559966767,0.9702957262759965),(0.20791169081775923,0.9781476007338057),(0.17364817766693041,0.984807753012208),(0.13917310096006547,0.9902680687415704),(0.10452846326765346,0.9945218953682733),(0.06975647374412523,0.9975640502598242),(0.03489949670250108,0.9993908270190958),(6.123233995736766e-17,1),(-0.03489949670250073,0.9993908270190958),(-0.06975647374412533,0.9975640502598242),(-0.10452846326765333,0.9945218953682734),(-0.13917310096006513,0.9902680687415704),(-0.1736481776669303,0.984807753012208),(-0.20791169081775912,0.9781476007338057),(-0.24192189559966756,0.9702957262759965),(-0.27563735581699905,0.9612616959383189),(-0.30901699437494734,0.9510565162951536),(-0.3420201433256687,0.9396926207859084),(-0.37460659341591207,0.9271838545667874),(-0.40673664307580004,0.913545457642601),(-0.4383711467890775,0.8987940462991669),(-0.46947156278589053,0.8829475928589271),(-0.4999999999999998,0.8660254037844387),(-0.5299192642332048,0.8480480961564261),(-0.5591929034707467,0.8290375725550417),(-0.587785252292473,0.8090169943749475),(-0.6156614753256583,0.788010753606722),(-0.6427876096865394,0.766044443118978),(-0.6691306063588579,0.7431448254773945),(-0.6946583704589974,0.7193398003386511),(-0.719339800338651,0.6946583704589975),(-0.7431448254773944,0.669130606358858),(-0.7660444431189779,0.6427876096865395),(-0.7880107536067219,0.6156614753256584),(-0.8090169943749473,0.5877852522924732),(-0.8290375725550416,0.5591929034707469),(-0.848048096156426,0.5299192642332049),(-0.8660254037844387,0.49999999999999994),(-0.882947592858927,0.4694715627858907),(-0.8987940462991668,0.4383711467890777),(-0.913545457642601,0.40673664307580004),(-0.9271838545667873,0.37460659341591224),(-0.9396926207859083,0.3420201433256689),(-0.9510565162951535,0.3090169943749475),(-0.9612616959383189,0.2756373558169992),(-0.9702957262759965,0.24192189559966773),(-0.9781476007338057,0.20791169081775931),(-0.984807753012208,0.1736481776669307),(-0.9902680687415704,0.13917310096006533),(-0.9945218953682733,0.10452846326765373),(-0.9975640502598242,0.06975647374412552),(-0.9993908270190958,0.03489949670250114),(-1,1.2246467991473532e-16),(-0.9993908270190958,-0.0348994967025009),(-0.9975640502598243,-0.06975647374412483),(-0.9945218953682733,-0.1045284632676535),(-0.9902680687415703,-0.13917310096006552),(-0.984807753012208,-0.17364817766693047),(-0.9781476007338057,-0.20791169081775907),(-0.9702957262759965,-0.2419218955996675),(-0.961261695938319,-0.2756373558169986),(-0.9510565162951535,-0.30901699437494773),(-0.9396926207859084,-0.34202014332566866),(-0.9271838545667874,-0.374606593415912),(-0.9135454576426011,-0.4067366430757998),(-0.8987940462991671,-0.43837114678907707),(-0.8829475928589271,-0.4694715627858905),(-0.8660254037844386,-0.5000000000000001),(-0.8480480961564261,-0.5299192642332048),(-0.8290375725550418,-0.5591929034707467),(-0.8090169943749476,-0.587785252292473),(-0.7880107536067222,-0.6156614753256578),(-0.766044443118978,-0.6427876096865393),(-0.7431448254773942,-0.6691306063588582),(-0.7193398003386511,-0.6946583704589974),(-0.6946583704589976,-0.7193398003386509),(-0.6691306063588585,-0.743144825477394),(-0.6427876096865395,-0.7660444431189779),(-0.6156614753256581,-0.7880107536067221),(-0.5877852522924732,-0.8090169943749473),(-0.5591929034707472,-0.8290375725550414),(-0.529919264233205,-0.848048096156426),(-0.5000000000000004,-0.8660254037844384),(-0.46947156278589075,-0.882947592858927),(-0.43837114678907774,-0.8987940462991668),(-0.4067366430758001,-0.913545457642601),(-0.3746065934159123,-0.9271838545667873),(-0.3420201433256694,-0.9396926207859082),(-0.30901699437494756,-0.9510565162951535),(-0.2756373558169989,-0.961261695938319),(-0.24192189559966779,-0.9702957262759965),(-0.2079116908177598,-0.9781476007338056),(-0.17364817766693033,-0.984807753012208),(-0.13917310096006583,-0.9902680687415703),(-0.10452846326765423,-0.9945218953682733),(-0.06975647374412558,-0.9975640502598242),(-0.03489949670250076,-0.9993908270190958),(-1.8369701987210297e-16,-1),(0.03489949670250039,-0.9993908270190958),(0.06975647374412522,-0.9975640502598243),(0.10452846326765387,-0.9945218953682733),(0.13917310096006547,-0.9902680687415704),(0.17364817766692997,-0.9848077530122081),(0.20791169081775943,-0.9781476007338056),(0.24192189559966745,-0.9702957262759966),(0.2756373558169985,-0.961261695938319),(0.30901699437494723,-0.9510565162951536),(0.342020143325669,-0.9396926207859083),(0.37460659341591196,-0.9271838545667874),(0.40673664307579976,-0.9135454576426011),(0.4383711467890774,-0.898794046299167),(0.4694715627858904,-0.8829475928589271),(0.5000000000000001,-0.8660254037844386),(0.5299192642332047,-0.8480480961564262),(0.559192903470747,-0.8290375725550416),(0.5877852522924729,-0.8090169943749476),(0.6156614753256578,-0.7880107536067223),(0.6427876096865393,-0.7660444431189781),(0.6691306063588585,-0.743144825477394),(0.6946583704589973,-0.7193398003386512),(0.7193398003386509,-0.6946583704589976),(0.7431448254773937,-0.6691306063588588),(0.7660444431189778,-0.6427876096865396),(0.788010753606722,-0.6156614753256582),(0.8090169943749473,-0.5877852522924734),(0.8290375725550414,-0.5591929034707473),(0.848048096156426,-0.529919264233205),(0.8660254037844384,-0.5000000000000004),(0.8829475928589269,-0.4694715627858908),(0.8987940462991668,-0.4383711467890778),(0.913545457642601,-0.40673664307580015),(0.9271838545667873,-0.37460659341591235),(0.9396926207859081,-0.34202014332566943),(0.9510565162951535,-0.3090169943749476),(0.9612616959383189,-0.27563735581699894),(0.9702957262759965,-0.24192189559966787),(0.9781476007338056,-0.20791169081775987),(0.984807753012208,-0.1736481776669304),(0.9902680687415703,-0.13917310096006588),(0.9945218953682733,-0.1045284632676543),(0.9975640502598242,-0.06975647374412564),(0.9993908270190958,-0.034899496702500823)]) FORMAT Null + + DROP TABLE IF EXISTS polygons + diff --git a/tests/performance/scripts/perf.py b/tests/performance/scripts/perf.py index 83d66997677..9931178fcb4 100755 --- a/tests/performance/scripts/perf.py +++ b/tests/performance/scripts/perf.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import argparse -import clickhouse_driver -import itertools import functools +import itertools +import logging import math import os import pprint @@ -14,9 +14,10 @@ import string import sys import time import traceback -import logging import xml.etree.ElementTree as et from threading import Thread + +import clickhouse_driver from scipy import stats logging.basicConfig( diff --git a/tests/queries/0_stateless/00240_replace_substring_loop.reference b/tests/queries/0_stateless/00240_replace_substring_loop.reference index 390ec161dc2..e32b5448f38 100644 --- a/tests/queries/0_stateless/00240_replace_substring_loop.reference +++ b/tests/queries/0_stateless/00240_replace_substring_loop.reference @@ -190,3 +190,6 @@ __.__ o_.__ o_.__ 1 __. o_. o_. 1 __.__ o_.__ o_.__ 1 __.__ o_.__ o_.__ 1 +ABCabc +ABCabc +ABCabc diff --git a/tests/queries/0_stateless/00240_replace_substring_loop.sql b/tests/queries/0_stateless/00240_replace_substring_loop.sql index 2c9157d5946..3757cc77395 100644 --- a/tests/queries/0_stateless/00240_replace_substring_loop.sql +++ b/tests/queries/0_stateless/00240_replace_substring_loop.sql @@ -99,3 +99,6 @@ SELECT s, replaceOne(s, '_', 'o') AS a, replaceRegexpOne(s, '_', 'o') AS b, a = SELECT s, replaceOne(s, '_', 'o') AS a, replaceRegexpOne(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '.__']) AS s); SELECT s, replaceOne(s, '_', 'o') AS a, replaceRegexpOne(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '__.']) AS s); SELECT s, replaceOne(s, '_', 'o') AS a, replaceRegexpOne(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '__.__']) AS s); +SELECT replace('ABCabc', '', 'DEF'); +SELECT replace(materialize('ABCabc'), materialize(''), 'DEF'); +SELECT replace(materialize('ABCabc'), '', 'DEF'); diff --git a/tests/queries/0_stateless/00386_long_in_pk.python b/tests/queries/0_stateless/00386_long_in_pk.python index c7b04102dc5..e8015284742 100644 --- a/tests/queries/0_stateless/00386_long_in_pk.python +++ b/tests/queries/0_stateless/00386_long_in_pk.python @@ -50,9 +50,10 @@ def gen_queries(): yield "select tuple(f) in (select tuple(tuple((a, b))) from tab_00386) from tab_00386" -import requests import os +import requests + def main(): url = os.environ["CLICKHOUSE_URL"] diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python index 38b108a696f..5f7f57a217e 100644 --- a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python +++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python @@ -1,6 +1,11 @@ #!/usr/bin/env python3 -import os, itertools, urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, sys +import itertools +import os +import sys +import urllib.error +import urllib.parse +import urllib.request CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/00646_url_engine.python b/tests/queries/0_stateless/00646_url_engine.python index 931d18a3f80..07e69d0a093 100644 --- a/tests/queries/0_stateless/00646_url_engine.python +++ b/tests/queries/0_stateless/00646_url_engine.python @@ -1,18 +1,18 @@ #!/usr/bin/env python3 -import socket import csv +import os +import socket +import subprocess import sys import tempfile import threading -import os import traceback -from urllib.parse import urljoin import urllib.request -import subprocess -from io import StringIO from http.server import BaseHTTPRequestHandler, HTTPServer +from io import StringIO from socketserver import ThreadingMixIn +from urllib.parse import urljoin def is_ipv6(host): diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.python b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python index 3db80aef845..503a0656082 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.python +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # encoding: utf-8 -import re -import itertools -import sys import argparse +import itertools +import re +import sys # Create SQL statement to verify dateTime64 is accepted as argument to functions taking DateTime. FUNCTIONS = """ diff --git a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py index e0f969050b5..0d808b3bcac 100755 --- a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py +++ b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py @@ -2,13 +2,13 @@ # Tags: no-parallel import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py index 33e63502ef2..88c9d340fc9 100755 --- a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py +++ b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py @@ -2,13 +2,13 @@ # Tags: no-parallel import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py index 22a171781d9..686a6702836 100755 --- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py +++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py index b9dacedd090..e552ad53b65 100755 --- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py +++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py @@ -7,7 +7,7 @@ import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py index 904723f650f..a8087a9994b 100755 --- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py +++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py @@ -2,13 +2,13 @@ # Tags: no-parallel, no-fasttest import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01070_window_view_watch_events.py b/tests/queries/0_stateless/01070_window_view_watch_events.py index 35f0155c624..79f7c884fe1 100755 --- a/tests/queries/0_stateless/01070_window_view_watch_events.py +++ b/tests/queries/0_stateless/01070_window_view_watch_events.py @@ -2,13 +2,13 @@ # Tags: no-parallel import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py index 275edb094c7..43d5348d895 100755 --- a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py +++ b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py @@ -7,7 +7,7 @@ import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py index 778ac786e88..bb938e78b06 100755 --- a/tests/queries/0_stateless/01082_window_view_watch_limit.py +++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py @@ -7,7 +7,7 @@ import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/01558_ttest_scipy.python b/tests/queries/0_stateless/01558_ttest_scipy.python index 75e1c2701b2..ee1cfa32038 100644 --- a/tests/queries/0_stateless/01558_ttest_scipy.python +++ b/tests/queries/0_stateless/01558_ttest_scipy.python @@ -1,9 +1,10 @@ #!/usr/bin/env python3 import os import sys -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/01561_mann_whitney_scipy.python b/tests/queries/0_stateless/01561_mann_whitney_scipy.python index 0f84d510933..83372324715 100644 --- a/tests/queries/0_stateless/01561_mann_whitney_scipy.python +++ b/tests/queries/0_stateless/01561_mann_whitney_scipy.python @@ -1,9 +1,10 @@ #!/usr/bin/env python3 import os import sys -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/01626_cnf_fuzz_long.python b/tests/queries/0_stateless/01626_cnf_fuzz_long.python index de9e4a21dbb..3105c24bc35 100644 --- a/tests/queries/0_stateless/01626_cnf_fuzz_long.python +++ b/tests/queries/0_stateless/01626_cnf_fuzz_long.python @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os -from random import randint, choices import sys +from random import choices, randint CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/01654_test_writer_block_sequence.python b/tests/queries/0_stateless/01654_test_writer_block_sequence.python index bc4e3da9ed5..fb861aabc7b 100644 --- a/tests/queries/0_stateless/01654_test_writer_block_sequence.python +++ b/tests/queries/0_stateless/01654_test_writer_block_sequence.python @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import os -import sys import random import string +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python index f363cb64018..ab6595c4a49 100644 --- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python +++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python @@ -1,8 +1,8 @@ -import pty +import multiprocessing import os +import pty import shlex import time -import multiprocessing COMPLETION_TIMEOUT_SECONDS = 30 DEBUG_LOG = os.path.join( diff --git a/tests/queries/0_stateless/01854_HTTP_dict_decompression.python b/tests/queries/0_stateless/01854_HTTP_dict_decompression.python index 7d98a24e83e..b7699b8c7e6 100644 --- a/tests/queries/0_stateless/01854_HTTP_dict_decompression.python +++ b/tests/queries/0_stateless/01854_HTTP_dict_decompression.python @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -from http.server import SimpleHTTPRequestHandler, HTTPServer -import socket import csv +import gzip +import lzma +import os +import socket +import subprocess import sys import tempfile import threading -import os -import gzip import traceback import urllib.request -import subprocess -import lzma +from http.server import HTTPServer, SimpleHTTPRequestHandler def is_ipv6(host): diff --git a/tests/queries/0_stateless/01921_test_progress_bar.py b/tests/queries/0_stateless/01921_test_progress_bar.py index 6406534a647..e686698ad9f 100755 --- a/tests/queries/0_stateless/01921_test_progress_bar.py +++ b/tests/queries/0_stateless/01921_test_progress_bar.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/02010_lc_native.python b/tests/queries/0_stateless/02010_lc_native.python index 6c4220855c8..4447750ef4d 100755 --- a/tests/queries/0_stateless/02010_lc_native.python +++ b/tests/queries/0_stateless/02010_lc_native.python @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import socket import os +import socket import uuid CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") diff --git a/tests/queries/0_stateless/02126_url_auth.python b/tests/queries/0_stateless/02126_url_auth.python index 9b2e68a017d..975a219f4ae 100644 --- a/tests/queries/0_stateless/02126_url_auth.python +++ b/tests/queries/0_stateless/02126_url_auth.python @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -import socket import csv +import os +import socket +import subprocess import sys import tempfile import threading -import os import traceback import urllib.request -import subprocess -from io import StringIO from http.server import BaseHTTPRequestHandler, HTTPServer +from io import StringIO def is_ipv6(host): diff --git a/tests/queries/0_stateless/02158_proportions_ztest_cmp.python b/tests/queries/0_stateless/02158_proportions_ztest_cmp.python index 0555f8c36ec..c74cd211f9f 100644 --- a/tests/queries/0_stateless/02158_proportions_ztest_cmp.python +++ b/tests/queries/0_stateless/02158_proportions_ztest_cmp.python @@ -1,11 +1,12 @@ #!/usr/bin/env python3 import os import sys -from math import sqrt, nan +from math import nan, sqrt from random import randrange -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02158_ztest_cmp.python b/tests/queries/0_stateless/02158_ztest_cmp.python index 9591a150337..e964a160cda 100644 --- a/tests/queries/0_stateless/02158_ztest_cmp.python +++ b/tests/queries/0_stateless/02158_ztest_cmp.python @@ -2,9 +2,10 @@ import os import sys from statistics import variance -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.python b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.python index 4c2df9556a1..4e33c88cd33 100644 --- a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.python +++ b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.python @@ -1,8 +1,8 @@ -import pty +import multiprocessing import os +import pty import shlex import time -import multiprocessing COMPLETION_TIMEOUT_SECONDS = 30 DEBUG_LOG = os.path.join( diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.python b/tests/queries/0_stateless/02205_HTTP_user_agent.python index 83089741bf2..13d3dc881ae 100644 --- a/tests/queries/0_stateless/02205_HTTP_user_agent.python +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.python @@ -1,14 +1,14 @@ #!/usr/bin/env python3 -from http.server import SimpleHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn +import os import socket +import subprocess import sys import threading -import os import traceback import urllib.request -import subprocess +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn def is_ipv6(host): diff --git a/tests/queries/0_stateless/02233_HTTP_ranged.python b/tests/queries/0_stateless/02233_HTTP_ranged.python index 5d06e4824b1..c4a584ae7ae 100644 --- a/tests/queries/0_stateless/02233_HTTP_ranged.python +++ b/tests/queries/0_stateless/02233_HTTP_ranged.python @@ -1,15 +1,15 @@ #!/usr/bin/env python3 -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn -import socket -import sys -import re -import threading import os +import re +import socket +import subprocess +import sys +import threading import traceback import urllib.request -import subprocess +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn def is_ipv6(host): diff --git a/tests/queries/0_stateless/02294_anova_cmp.python b/tests/queries/0_stateless/02294_anova_cmp.python index 2212a887b2f..1646d03f805 100644 --- a/tests/queries/0_stateless/02294_anova_cmp.python +++ b/tests/queries/0_stateless/02294_anova_cmp.python @@ -2,9 +2,10 @@ import os import sys from statistics import variance -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02403_big_http_chunk_size.python b/tests/queries/0_stateless/02403_big_http_chunk_size.python index f74459489a5..a0f0fd474e3 100644 --- a/tests/queries/0_stateless/02403_big_http_chunk_size.python +++ b/tests/queries/0_stateless/02403_big_http_chunk_size.python @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -from socket import socket, AF_INET, SOCK_STREAM import os - +from socket import AF_INET, SOCK_STREAM, socket EXCEPTION_CODE_HEADER = "X-ClickHouse-Exception-Code" TRANSFER_ENCODING_HEADER = "Transfer-Encoding" diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 205e772bbbb..7c541f272c8 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -321,7 +321,6 @@ geohashesInBox getMacro getOSKernelVersion getServerPort -getSetting getSizeOfEnumType getTypeSerializationStreams globalIn diff --git a/tests/queries/0_stateless/02423_insert_stats_behaviour.sh b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh index b85ca311101..5680af7da71 100755 --- a/tests/queries/0_stateless/02423_insert_stats_behaviour.sh +++ b/tests/queries/0_stateless/02423_insert_stats_behaviour.sh @@ -4,9 +4,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple();" -$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple();" -$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0;" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0;" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target TO target_1 AS SELECT * FROM floats" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target_2 TO target_2 AS SELECT * FROM floats, numbers(2) n" diff --git a/tests/queries/0_stateless/02423_insert_summary_behaviour.sh b/tests/queries/0_stateless/02423_insert_summary_behaviour.sh index b184d9ccf47..cb28724ab58 100755 --- a/tests/queries/0_stateless/02423_insert_summary_behaviour.sh +++ b/tests/queries/0_stateless/02423_insert_summary_behaviour.sh @@ -4,9 +4,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple();" -$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple();" -$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple();" +$CLICKHOUSE_CLIENT -q "CREATE TABLE floats (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0;" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_1 (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0;" +$CLICKHOUSE_CLIENT -q "CREATE TABLE target_2 (v Float64) Engine=MergeTree() ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0;" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target TO target_1 AS SELECT * FROM floats" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW floats_to_target_2 TO target_2 AS SELECT * FROM floats, numbers(2) n" diff --git a/tests/queries/0_stateless/02437_drop_mv_restart_replicas.sh b/tests/queries/0_stateless/02437_drop_mv_restart_replicas.sh index 44076aeba18..af6d15b60e6 100755 --- a/tests/queries/0_stateless/02437_drop_mv_restart_replicas.sh +++ b/tests/queries/0_stateless/02437_drop_mv_restart_replicas.sh @@ -53,7 +53,7 @@ export -f thread_restart; TIMEOUT=15 -timeout $TIMEOUT bash -c thread_ddl 2>&1| grep -Fa "Exception: " | grep -Fv -e "TABLE_IS_DROPPED" -e "UNKNOWN_TABLE" -e "DATABASE_NOT_EMPTY" & +timeout $TIMEOUT bash -c thread_ddl 2>&1| grep -Fa "Exception: " | grep -Fv -e "TABLE_IS_DROPPED" -e "UNKNOWN_TABLE" -e "DATABASE_NOT_EMPTY" -e "TABLE_IS_BEING_RESTARTED" & timeout $TIMEOUT bash -c thread_insert 2> /dev/null & timeout $TIMEOUT bash -c thread_restart 2>&1| grep -Fa "Exception: " | grep -Fv -e "is currently dropped or renamed" -e "is being dropped or detached" & diff --git a/tests/queries/0_stateless/02458_insert_select_progress_tcp.python b/tests/queries/0_stateless/02458_insert_select_progress_tcp.python index 92240e109c1..01622555f48 100644 --- a/tests/queries/0_stateless/02458_insert_select_progress_tcp.python +++ b/tests/queries/0_stateless/02458_insert_select_progress_tcp.python @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import socket -import os -import uuid import json +import os +import socket +import uuid CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000")) diff --git a/tests/queries/0_stateless/02473_infile_progress.py b/tests/queries/0_stateless/02473_infile_progress.py index 4165eeb6d31..b1303feb5ed 100755 --- a/tests/queries/0_stateless/02473_infile_progress.py +++ b/tests/queries/0_stateless/02473_infile_progress.py @@ -2,13 +2,13 @@ # Tags: no-replicated-database, no-parallel, no-fasttest import os -import sys import signal +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) -from client import client, prompt, end_of_block +from client import client, end_of_block, prompt log = None # uncomment the line below for debugging diff --git a/tests/queries/0_stateless/02473_multistep_prewhere.python b/tests/queries/0_stateless/02473_multistep_prewhere.python index 09326b6365d..35d00fb8cfe 100644 --- a/tests/queries/0_stateless/02473_multistep_prewhere.python +++ b/tests/queries/0_stateless/02473_multistep_prewhere.python @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import requests import os import sys +import requests + CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02473_multistep_split_prewhere.python b/tests/queries/0_stateless/02473_multistep_split_prewhere.python index 10e94059171..fa5b48c020a 100644 --- a/tests/queries/0_stateless/02473_multistep_split_prewhere.python +++ b/tests/queries/0_stateless/02473_multistep_split_prewhere.python @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import requests import os import sys +import requests + CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02481_async_insert_dedup.python b/tests/queries/0_stateless/02481_async_insert_dedup.python index c8b5abc11b0..dd0aa70b593 100644 --- a/tests/queries/0_stateless/02481_async_insert_dedup.python +++ b/tests/queries/0_stateless/02481_async_insert_dedup.python @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os -import sys -import random import queue +import random +import sys import time from threading import Thread diff --git a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference index 5e50b9e6cbf..de65b4fa268 100644 --- a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference +++ b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference @@ -134,4 +134,16 @@ 3 Hello World not_found x Hello World 4 Hello World [eo] x Hxllo World 5 Hello World . x xello World -Check that an exception is thrown if the needle is empty +Check that whether an exception is thrown if the needle is empty +Hexxo Worxd +Hello World +Hexlo World +Hello World +Hello World +Hello World +Hello World +Hello World +Hexxo Worxd +Hello World +Hexlo World +Hello World diff --git a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql index b88224a89c1..bccd9f3e609 100644 --- a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql +++ b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql @@ -70,7 +70,7 @@ SELECT id, haystack, needle, replacement, replaceRegexpOne('Hello World', needle DROP TABLE IF EXISTS test_tab; -SELECT 'Check that an exception is thrown if the needle is empty'; +SELECT 'Check that whether an exception is thrown if the needle is empty'; CREATE TABLE test_tab (id UInt32, haystack String, needle String, replacement String) @@ -80,20 +80,20 @@ CREATE TABLE test_tab INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'x') (2, 'Hello World', '', 'y'); -- needle: non-const, replacement: const -SELECT replaceAll(haystack, needle, 'x') FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT replaceOne(haystack, needle, 'x') FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT replaceAll(haystack, needle, 'x') FROM test_tab; +SELECT replaceOne(haystack, needle, 'x') FROM test_tab; SELECT replaceRegexpAll(haystack, needle, 'x') FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT replaceRegexpOne(haystack, needle, 'x') FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } -- needle: const, replacement: non-const -SELECT replaceAll(haystack, '', replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT replaceOne(haystack, '', replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT replaceAll(haystack, '', replacement) FROM test_tab; +SELECT replaceOne(haystack, '', replacement) FROM test_tab; SELECT replaceRegexpAll(haystack, '', replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT replaceRegexpOne(haystack, '', replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } -- needle: non-const, replacement: non-const -SELECT replaceAll(haystack, needle, replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } -SELECT replaceOne(haystack, needle, replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT replaceAll(haystack, needle, replacement) FROM test_tab; +SELECT replaceOne(haystack, needle, replacement) FROM test_tab; SELECT replaceRegexpAll(haystack, needle, replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } SELECT replaceRegexpOne(haystack, needle, replacement) FROM test_tab; -- { serverError ARGUMENT_OUT_OF_BOUND } diff --git a/tests/queries/0_stateless/02597_column_update_tricky_expression_and_replication.python b/tests/queries/0_stateless/02597_column_update_tricky_expression_and_replication.python index eb0cab9d56f..75cb3381bfd 100644 --- a/tests/queries/0_stateless/02597_column_update_tricky_expression_and_replication.python +++ b/tests/queries/0_stateless/02597_column_update_tricky_expression_and_replication.python @@ -2,15 +2,14 @@ import os import sys -from threading import Thread from queue import Queue +from threading import Thread CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) from pure_http_client import ClickHouseClient - client = ClickHouseClient() diff --git a/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python b/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python index 01f245e0cf0..33e2f3f0e02 100644 --- a/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python +++ b/tests/queries/0_stateless/02706_kolmogorov_smirnov_test_scipy.python @@ -1,9 +1,10 @@ #!/usr/bin/env python3 import os import sys -from scipy import stats -import pandas as pd + import numpy as np +import pandas as pd +from scipy import stats CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python index 48b27d434ec..06583f4e6b0 100644 --- a/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python +++ b/tests/queries/0_stateless/02750_settings_alias_tcp_protocol.python @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import socket -import os -import uuid import json +import os +import socket +import uuid CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000")) diff --git a/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.reference b/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.reference new file mode 100644 index 00000000000..06904000472 --- /dev/null +++ b/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.reference @@ -0,0 +1,6 @@ +0 +1 1 +1 2 +1 3 +1 4 +2 2000 diff --git a/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.sh b/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.sh new file mode 100755 index 00000000000..857ee521ce4 --- /dev/null +++ b/tests/queries/0_stateless/02861_alter_replace_partition_do_not_wait_mutations_on_unrelated_partitions.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# https://github.com/ClickHouse/ClickHouse/issues/45328 +# Check that replacing one partition on a table with `ALTER TABLE REPLACE PARTITION` +# doesn't wait for mutations on other partitions. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t1;" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t2;" +$CLICKHOUSE_CLIENT -q " +CREATE TABLE t1 +( + p UInt8, + i UInt64 +) +ENGINE = MergeTree +PARTITION BY p +ORDER BY tuple(); +" + + +$CLICKHOUSE_CLIENT -q "INSERT INTO t1 VALUES (1, 1), (1, 2), (1, 3), (1, 4), (2, 5);" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t2 AS t1;" +$CLICKHOUSE_CLIENT -q "INSERT INTO t2 VALUES (2, 2000);" + +# mutation that is supposed to be running in background while REPLACE is performed. +# sleepEachRow(3) is causing a mutation on partition 1 to be stuck. We test that another mutation on an unrelated partition will not wait for this one. +$CLICKHOUSE_CLIENT -q "ALTER TABLE t1 UPDATE i = sleepEachRow(3) IN PARTITION id '1' WHERE p == 1;" + +# wait for mutation to start +while [ "$($CLICKHOUSE_CLIENT -q "SELECT is_done as is_running FROM system.mutations WHERE database==currentDatabase() AND table=='t1'")" != 0 ] +do + sleep .5 +done + +# Run mutation on another partition +$CLICKHOUSE_CLIENT -q "ALTER TABLE t1 REPLACE PARTITION id '2' FROM t2 SETTINGS mutations_sync=2;" + +# check that mutation is still running +$CLICKHOUSE_CLIENT -q "SELECT is_done FROM system.mutations WHERE database==currentDatabase() AND table=='t1';" + +$CLICKHOUSE_CLIENT -q "SELECT * FROM t1 ORDER BY i;" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t1" +$CLICKHOUSE_CLIENT -q "DROP TABLE t2" diff --git a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.reference b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.reference index 637bed86848..a1a206efdf6 100644 --- a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.reference +++ b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.reference @@ -29,6 +29,7 @@ OK 100 100 OK +Syntax error ===== TestGrants ===== OK OK diff --git a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh index 8409f4b0b7c..6f0acb22395 100755 --- a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh +++ b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh @@ -208,6 +208,14 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM destination2" (( $(${CLICKHOUSE_CLIENT} --query "ALTER TABLE test_table MODIFY SQL SECURITY INVOKER" 2>&1 | grep -c "is not supported") >= 1 )) && echo "OK" || echo "UNEXPECTED" +(( $(${CLICKHOUSE_CLIENT} --user $user1 --query " + CREATE VIEW $db.test_view_broken + SQL SECURITY DEFINER + DEFINER CURRENT_USER + DEFINER $user2 + AS SELECT * FROM $db.test_table; +" 2>&1 | grep -c "Syntax error") >= 1 )) && echo "Syntax error" || echo "UNEXPECTED" + echo "===== TestGrants =====" ${CLICKHOUSE_CLIENT} --query "GRANT CREATE ON *.* TO $user1" ${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON $db.test_table TO $user1, $user2" diff --git a/tests/queries/0_stateless/02933_sqid.reference b/tests/queries/0_stateless/02933_sqid.reference index 4597e2347e3..71964e69e3c 100644 --- a/tests/queries/0_stateless/02933_sqid.reference +++ b/tests/queries/0_stateless/02933_sqid.reference @@ -5,6 +5,7 @@ XMbT [1,2] 86Rf07 [1,2,3] Td1EnWQo [1,2,3,4] XMbT +[] -- non-const UInt* Uk [1] XMbT [1,2] diff --git a/tests/queries/0_stateless/02933_sqid.sql b/tests/queries/0_stateless/02933_sqid.sql index 822fe33df51..3dcca13a8c4 100644 --- a/tests/queries/0_stateless/02933_sqid.sql +++ b/tests/queries/0_stateless/02933_sqid.sql @@ -14,6 +14,7 @@ SELECT sqidEncode(1, 2) AS sqid, sqidDecode(sqid); SELECT sqidEncode(1, 2, 3) AS sqid, sqidDecode(sqid); SELECT sqidEncode(1::UInt8, 2::UInt16, 3::UInt32, 4::UInt64) AS sqid, sqidDecode(sqid); SELECT sqidEncode(toNullable(1), toLowCardinality(2)) AS sqid; +SELECT sqidDecode('1'); SELECT '-- non-const UInt*'; SELECT sqidEncode(materialize(1)) AS sqid, sqidDecode(sqid); diff --git a/tests/queries/0_stateless/03008_deduplication.python b/tests/queries/0_stateless/03008_deduplication.python index bfaa496805f..d38b6d0b716 100644 --- a/tests/queries/0_stateless/03008_deduplication.python +++ b/tests/queries/0_stateless/03008_deduplication.python @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -import os -import sys import argparse +import os import string - +import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(CURDIR, "helpers")) diff --git a/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.reference b/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.reference new file mode 100644 index 00000000000..f1c34569326 --- /dev/null +++ b/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.reference @@ -0,0 +1,8 @@ +4 4 +6 6 +8 8 + "rows_read": 3, +12 12 +14 14 + "rows_read": 3, + "rows_read": 0, diff --git a/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.sh b/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.sh new file mode 100755 index 00000000000..1502be848c2 --- /dev/null +++ b/tests/queries/0_stateless/03031_minmax_index_for_pointinpolygon.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists minmax_index_point_in_polygon" +$CLICKHOUSE_CLIENT -q " +CREATE TABLE minmax_index_point_in_polygon +( + x UInt32, + y UInt32, + INDEX mm_x_y (x, y) TYPE minmax GRANULARITY 1 +) +ENGINE = MergeTree() +ORDER BY x +SETTINGS index_granularity = 3" + +$CLICKHOUSE_CLIENT -q "Insert into minmax_index_point_in_polygon values + (4, 4), + (6 ,6), + (8, 8), + (12, 12), + (14, 14), + (16, 16)" + +function query_and_check() +{ + query="$1" + $CLICKHOUSE_CLIENT -q "$query" + $CLICKHOUSE_CLIENT -q "$query FORMAT JSON" | grep "rows_read" +} + +# 1/2 marks filtered by minmax index, read_rows should be 3 +query_and_check "select * from minmax_index_point_in_polygon where pointInPolygon((x, y), [(4., 4.), (8., 4.), (8., 8.), (4., 8.)])" +query_and_check "select * from minmax_index_point_in_polygon where pointInPolygon((x, y), [(10., 13.), (14., 14.), (14, 10)])" + + +# 2/2 marks filtered by minmax index, read_rows should be 0 +query_and_check "select * from minmax_index_point_in_polygon where pointInPolygon((x, y), [(0., 0.), (2., 2.), (2., 0.)])" + +$CLICKHOUSE_CLIENT -q "drop table if exists minmax_index_point_in_polygon" diff --git a/tests/queries/0_stateless/03215_grant_current_grants.reference b/tests/queries/0_stateless/03215_grant_current_grants.reference index e4f6850b806..2e58b8a82a3 100644 --- a/tests/queries/0_stateless/03215_grant_current_grants.reference +++ b/tests/queries/0_stateless/03215_grant_current_grants.reference @@ -1,2 +1,5 @@ GRANT SELECT, CREATE TABLE, CREATE VIEW ON default.* GRANT SELECT ON default.* +OK +GRANT SELECT ON *.* +REVOKE SELECT ON system.zookeeper diff --git a/tests/queries/0_stateless/03215_grant_current_grants.sh b/tests/queries/0_stateless/03215_grant_current_grants.sh index 68af4a62bba..07981d23fbe 100755 --- a/tests/queries/0_stateless/03215_grant_current_grants.sh +++ b/tests/queries/0_stateless/03215_grant_current_grants.sh @@ -11,6 +11,7 @@ user3="user03215_3_${CLICKHOUSE_DATABASE}_$RANDOM" db=${CLICKHOUSE_DATABASE} +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS $user1, $user2, $user3"; ${CLICKHOUSE_CLIENT} --query "CREATE USER $user1, $user2, $user3;"; ${CLICKHOUSE_CLIENT} --query "GRANT SELECT, CREATE TABLE, CREATE VIEW ON $db.* TO $user1 WITH GRANT OPTION;"; @@ -23,4 +24,14 @@ ${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR $user3" | sed 's/ TO.*//'; ${CLICKHOUSE_CLIENT} --query "GRANT CURRENT GRANTS(SELECT ON $db.*) TO $user3" --user $user1; ${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR $user3" | sed 's/ TO.*//'; +${CLICKHOUSE_CLIENT} --query "REVOKE ALL ON *.* FROM $user1"; +${CLICKHOUSE_CLIENT} --query "REVOKE ALL ON *.* FROM $user2"; + +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON *.* TO $user1 WITH GRANT OPTION"; +${CLICKHOUSE_CLIENT} --query "REVOKE SELECT ON system.zookeeper FROM $user1"; + +(( $(${CLICKHOUSE_CLIENT} --user $user1 --query "GRANT SELECT ON *.* TO $user2" 2>&1 | grep -c "(Missing permissions: SELECT ON system.*)") >= 1 )) && echo "OK" || echo "UNEXPECTED" +${CLICKHOUSE_CLIENT} --query "GRANT CURRENT GRANTS(SELECT ON *.*) TO $user2" --user $user1; +${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR $user2" | sed 's/ TO.*//' | sed 's/ FROM.*//'; + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS $user1, $user2, $user3"; diff --git a/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.reference b/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.reference deleted file mode 100644 index c563617a01c..00000000000 --- a/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.reference +++ /dev/null @@ -1,40 +0,0 @@ -CREATE TABLE default.uk_price_paid\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid -CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month -CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE default.uk_price_paid\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid -CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month -CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE `default`.`uk_price_paid`\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX `county_index` `county` TYPE set(10) GRANULARITY 1,\n PROJECTION `town_date_projection`\n (\n SELECT \n `town`,\n `date`,\n `price`\n ORDER BY \n `town`,\n `date`\n ),\n PROJECTION `handy_aggs_projection`\n (\n SELECT \n avg(`price`),\n max(`price`),\n sum(`price`)\n GROUP BY `town`\n )\n)\nENGINE = MergeTree\nORDER BY (`postcode1`, `postcode2`, `date`)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW `default`.`prices_by_year_view` TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n `price`,\n `date`,\n `addr1`,\n `addr2`,\n `street`,\n `town`,\n `district`,\n `county`\nFROM `default`.`uk_price_paid` -CREATE TABLE `default`.`uk_prices_aggs_dest`\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW `default`.`uk_prices_aggs_view` TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(`price`) AS `min_price`,\n maxSimpleState(`price`) AS `max_price`,\n countState(`price`) AS `volume`,\n avgState(`price`) AS `avg_price`\nFROM `default`.`uk_price_paid`\nGROUP BY `month` -CREATE DICTIONARY `default`.`uk_mortgage_rates_dict`\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE default.uk_price_paid\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid -CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month -CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE "default"."uk_price_paid"\n(\n "price" UInt32,\n "date" Date,\n "postcode1" LowCardinality(String),\n "postcode2" LowCardinality(String),\n "type" Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n "is_new" UInt8,\n "duration" Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "locality" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String),\n INDEX "county_index" "county" TYPE set(10) GRANULARITY 1,\n PROJECTION "town_date_projection"\n (\n SELECT \n "town",\n "date",\n "price"\n ORDER BY \n "town",\n "date"\n ),\n PROJECTION "handy_aggs_projection"\n (\n SELECT \n avg("price"),\n max("price"),\n sum("price")\n GROUP BY "town"\n )\n)\nENGINE = MergeTree\nORDER BY ("postcode1", "postcode2", "date")\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW "default"."prices_by_year_view" TO default.prices_by_year_dest\n(\n "price" UInt32,\n "date" Date,\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String)\n)\nAS SELECT\n "price",\n "date",\n "addr1",\n "addr2",\n "street",\n "town",\n "district",\n "county"\nFROM "default"."uk_price_paid" -CREATE TABLE "default"."uk_prices_aggs_dest"\n(\n "month" Date,\n "min_price" SimpleAggregateFunction("min", UInt32),\n "max_price" SimpleAggregateFunction("max", UInt32),\n "volume" AggregateFunction("count", UInt32),\n "avg_price" AggregateFunction("avg", UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY "month"\nORDER BY "month"\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW "default"."uk_prices_aggs_view" TO default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction("min", UInt32),\n "max_price" SimpleAggregateFunction("max", UInt32),\n "volume" AggregateFunction("count", UInt32),\n "avg_price" AggregateFunction("avg", UInt32)\n)\nAS WITH toStartOfMonth("date") AS "month"\nSELECT\n "month",\n minSimpleState("price") AS "min_price",\n maxSimpleState("price") AS "max_price",\n countState("price") AS "volume",\n avgState("price") AS "avg_price"\nFROM "default"."uk_price_paid"\nGROUP BY "month" -CREATE DICTIONARY "default"."uk_mortgage_rates_dict"\n(\n "date" DateTime64,\n "variable" Decimal32(2),\n "fixed" Decimal32(2),\n "bank" Decimal32(2)\n)\nPRIMARY KEY "date"\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE default.uk_price_paid\n(\n "price" UInt32,\n "date" Date,\n "postcode1" LowCardinality(String),\n "postcode2" LowCardinality(String),\n "type" Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n "is_new" UInt8,\n "duration" Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "locality" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n "price" UInt32,\n "date" Date,\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid -CREATE TABLE default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction(min, UInt32),\n "max_price" SimpleAggregateFunction(max, UInt32),\n "volume" AggregateFunction(count, UInt32),\n "avg_price" AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction(min, UInt32),\n "max_price" SimpleAggregateFunction(max, UInt32),\n "volume" AggregateFunction(count, UInt32),\n "avg_price" AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month -CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n "date" DateTime64,\n "variable" Decimal32(2),\n "fixed" Decimal32(2),\n "bank" Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE `default`.`uk_price_paid`\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX `county_index` `county` TYPE set(10) GRANULARITY 1,\n PROJECTION `town_date_projection`\n (\n SELECT \n `town`,\n `date`,\n `price`\n ORDER BY \n `town`,\n `date`\n ),\n PROJECTION `handy_aggs_projection`\n (\n SELECT \n avg(`price`),\n max(`price`),\n sum(`price`)\n GROUP BY `town`\n )\n)\nENGINE = MergeTree\nORDER BY (`postcode1`, `postcode2`, `date`)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW `default`.`prices_by_year_view` TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n `price`,\n `date`,\n `addr1`,\n `addr2`,\n `street`,\n `town`,\n `district`,\n `county`\nFROM `default`.`uk_price_paid` -CREATE TABLE `default`.`uk_prices_aggs_dest`\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW `default`.`uk_prices_aggs_view` TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(`price`) AS `min_price`,\n maxSimpleState(`price`) AS `max_price`,\n countState(`price`) AS `volume`,\n avgState(`price`) AS `avg_price`\nFROM `default`.`uk_price_paid`\nGROUP BY `month` -CREATE DICTIONARY `default`.`uk_mortgage_rates_dict`\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) -CREATE TABLE default.uk_price_paid\n(\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid -CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 -CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month -CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) diff --git a/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.sql b/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.sql deleted file mode 100644 index c500dd4e4c6..00000000000 --- a/tests/queries/0_stateless/03230_output_format_identifier_quoting_style.sql +++ /dev/null @@ -1,328 +0,0 @@ -DROP DICTIONARY IF EXISTS uk_mortgage_rates_dict; -DROP TABLE IF EXISTS uk_mortgage_rates; -DROP VIEW IF EXISTS uk_prices_aggs_view; -DROP TABLE IF EXISTS uk_prices_aggs_dest; -DROP VIEW IF EXISTS prices_by_year_view; -DROP TABLE IF EXISTS prices_by_year_dest; -DROP TABLE IF EXISTS uk_price_paid; - --- Create tables, views, dictionaries - -CREATE TABLE uk_price_paid -( - price UInt32, - date Date, - postcode1 LowCardinality(String), - postcode2 LowCardinality(String), - type Enum('terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4, 'other' = 0), - is_new UInt8, - duration Enum('freehold' = 1, 'leasehold' = 2, 'unknown' = 0), - addr1 String, - addr2 String, - street LowCardinality(String), - locality LowCardinality(String), - town LowCardinality(String), - district LowCardinality(String), - county LowCardinality(String), - INDEX county_index county TYPE set(10) GRANULARITY 1, - PROJECTION town_date_projection - ( - SELECT - town, - date, - price - ORDER BY - town, - date - ), - PROJECTION handy_aggs_projection - ( - SELECT - avg(price), - max(price), - sum(price) - GROUP BY town - ) -) -ENGINE = MergeTree -ORDER BY (postcode1, postcode2, date); - -CREATE TABLE prices_by_year_dest ( - price UInt32, - date Date, - addr1 String, - addr2 String, - street LowCardinality(String), - town LowCardinality(String), - district LowCardinality(String), - county LowCardinality(String) -) -ENGINE = MergeTree -PRIMARY KEY (town, date) -PARTITION BY toYear(date); - -CREATE MATERIALIZED VIEW prices_by_year_view -TO prices_by_year_dest -AS - SELECT - price, - date, - addr1, - addr2, - street, - town, - district, - county - FROM uk_price_paid; - -CREATE TABLE uk_prices_aggs_dest ( - month Date, - min_price SimpleAggregateFunction(min, UInt32), - max_price SimpleAggregateFunction(max, UInt32), - volume AggregateFunction(count, UInt32), - avg_price AggregateFunction(avg, UInt32) -) -ENGINE = AggregatingMergeTree -PRIMARY KEY month; - -CREATE MATERIALIZED VIEW uk_prices_aggs_view -TO uk_prices_aggs_dest -AS - WITH - toStartOfMonth(date) AS month - SELECT - month, - minSimpleState(price) AS min_price, - maxSimpleState(price) AS max_price, - countState(price) AS volume, - avgState(price) AS avg_price - FROM uk_price_paid - GROUP BY month; - -CREATE TABLE uk_mortgage_rates ( - date DateTime64, - variable Decimal32(2), - fixed Decimal32(2), - bank Decimal32(2) -) -ENGINE Memory(); - -INSERT INTO uk_mortgage_rates VALUES ('2004-02-29', 5.02, 4.9, 4); -INSERT INTO uk_mortgage_rates VALUES ('2004-03-31', 5.11, 4.91, 4); - -CREATE DICTIONARY uk_mortgage_rates_dict ( - date DateTime64, - variable Decimal32(2), - fixed Decimal32(2), - bank Decimal32(2) -) -PRIMARY KEY date -SOURCE( - CLICKHOUSE(TABLE 'uk_mortgage_rates') -) -LAYOUT(COMPLEX_KEY_HASHED()) -LIFETIME(2628000000); - - --- Show tables, views, dictionaries with default settings -SHOW CREATE TABLE uk_price_paid; - -SHOW CREATE VIEW prices_by_year_view; - -SHOW CREATE uk_prices_aggs_dest; - -SHOW CREATE VIEW uk_prices_aggs_view; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict; - - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=false, output_format_identifier_quoting_style='None' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='None'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='None'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='None'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='None'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='None'; - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=true, output_format_identifier_quoting_style='Backticks' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='Backticks'; - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=false, output_format_identifier_quoting_style='Backticks' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='Backticks'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='Backticks'; - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=true, output_format_identifier_quoting_style='DoubleQuotes' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='DoubleQuotes'; - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=false, output_format_identifier_quoting_style='DoubleQuotes' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='DoubleQuotes'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='DoubleQuotes'; - - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=true, output_format_identifier_quoting_style='BackticksMySQL' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=true, - output_format_identifier_quoting_style='BackticksMySQL'; - --- Show tables, views, dictionaries with output_format_always_quote_identifiers=false, output_format_identifier_quoting_style='BackticksMySQL' -SHOW CREATE TABLE uk_price_paid -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE VIEW prices_by_year_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE uk_prices_aggs_dest -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE VIEW uk_prices_aggs_view -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='BackticksMySQL'; - -SHOW CREATE DICTIONARY uk_mortgage_rates_dict -SETTINGS - output_format_always_quote_identifiers=false, - output_format_identifier_quoting_style='BackticksMySQL'; - -DROP DICTIONARY uk_mortgage_rates_dict; -DROP TABLE uk_mortgage_rates; -DROP VIEW uk_prices_aggs_view; -DROP TABLE uk_prices_aggs_dest; -DROP VIEW prices_by_year_view; -DROP TABLE prices_by_year_dest; -DROP TABLE uk_price_paid; diff --git a/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.reference b/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.reference new file mode 100644 index 00000000000..c0e3dc474d8 --- /dev/null +++ b/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.reference @@ -0,0 +1,60 @@ +Settings: default +CREATE TABLE default.uk_price_paid\n(\n `Table` String,\n `Engine` String,\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: always & Backticks +CREATE TABLE `default`.`uk_price_paid`\n(\n `Table` String,\n `Engine` String,\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX `county_index` `county` TYPE set(10) GRANULARITY 1,\n PROJECTION `town_date_projection`\n (\n SELECT \n `town`,\n `date`,\n `price`\n ORDER BY \n `town`,\n `date`\n ),\n PROJECTION `handy_aggs_projection`\n (\n SELECT \n avg(`price`),\n max(`price`),\n sum(`price`)\n GROUP BY `town`\n )\n)\nENGINE = MergeTree\nORDER BY (`postcode1`, `postcode2`, `date`)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW `default`.`prices_by_year_view` TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n `price`,\n `date`,\n `addr1`,\n `addr2`,\n `street`,\n `town`,\n `district`,\n `county`\nFROM `default`.`uk_price_paid` +CREATE TABLE `default`.`uk_prices_aggs_dest`\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW `default`.`uk_prices_aggs_view` TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(`price`) AS `min_price`,\n maxSimpleState(`price`) AS `max_price`,\n countState(`price`) AS `volume`,\n avgState(`price`) AS `avg_price`\nFROM `default`.`uk_price_paid`\nGROUP BY `month` +CREATE DICTIONARY `default`.`uk_mortgage_rates_dict`\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: user_display & Backticks +CREATE TABLE default.uk_price_paid\n(\n `Table` String,\n `Engine` String,\n price UInt32,\n `date` Date,\n postcode1 LowCardinality(String),\n postcode2 LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n is_new UInt8,\n duration Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n locality LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n `date`,\n price\n ORDER BY \n town,\n `date`\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, `date`)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n price UInt32,\n `date` Date,\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String)\n)\nAS SELECT\n price,\n `date`,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n min_price SimpleAggregateFunction(`min`, UInt32),\n max_price SimpleAggregateFunction(`max`, UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n min_price SimpleAggregateFunction(`min`, UInt32),\n max_price SimpleAggregateFunction(`max`, UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY `month` +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n variable Decimal32(2),\n fixed Decimal32(2),\n bank Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: when_necessary & Backticks +CREATE TABLE default.uk_price_paid\n(\n `Table` String,\n `Engine` String,\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: always & DoubleQuotes +CREATE TABLE "default"."uk_price_paid"\n(\n "Table" String,\n "Engine" String,\n "price" UInt32,\n "date" Date,\n "postcode1" LowCardinality(String),\n "postcode2" LowCardinality(String),\n "type" Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n "is_new" UInt8,\n "duration" Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "locality" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String),\n INDEX "county_index" "county" TYPE set(10) GRANULARITY 1,\n PROJECTION "town_date_projection"\n (\n SELECT \n "town",\n "date",\n "price"\n ORDER BY \n "town",\n "date"\n ),\n PROJECTION "handy_aggs_projection"\n (\n SELECT \n avg("price"),\n max("price"),\n sum("price")\n GROUP BY "town"\n )\n)\nENGINE = MergeTree\nORDER BY ("postcode1", "postcode2", "date")\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW "default"."prices_by_year_view" TO default.prices_by_year_dest\n(\n "price" UInt32,\n "date" Date,\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String)\n)\nAS SELECT\n "price",\n "date",\n "addr1",\n "addr2",\n "street",\n "town",\n "district",\n "county"\nFROM "default"."uk_price_paid" +CREATE TABLE "default"."uk_prices_aggs_dest"\n(\n "month" Date,\n "min_price" SimpleAggregateFunction("min", UInt32),\n "max_price" SimpleAggregateFunction("max", UInt32),\n "volume" AggregateFunction("count", UInt32),\n "avg_price" AggregateFunction("avg", UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY "month"\nORDER BY "month"\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW "default"."uk_prices_aggs_view" TO default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction("min", UInt32),\n "max_price" SimpleAggregateFunction("max", UInt32),\n "volume" AggregateFunction("count", UInt32),\n "avg_price" AggregateFunction("avg", UInt32)\n)\nAS WITH toStartOfMonth("date") AS "month"\nSELECT\n "month",\n minSimpleState("price") AS "min_price",\n maxSimpleState("price") AS "max_price",\n countState("price") AS "volume",\n avgState("price") AS "avg_price"\nFROM "default"."uk_price_paid"\nGROUP BY "month" +CREATE DICTIONARY "default"."uk_mortgage_rates_dict"\n(\n "date" DateTime64,\n "variable" Decimal32(2),\n "fixed" Decimal32(2),\n "bank" Decimal32(2)\n)\nPRIMARY KEY "date"\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: user_display & DoubleQuotes +CREATE TABLE default.uk_price_paid\n(\n "Table" String,\n "Engine" String,\n price UInt32,\n "date" Date,\n postcode1 LowCardinality(String),\n postcode2 LowCardinality(String),\n "type" Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n is_new UInt8,\n duration Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n locality LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n "date",\n price\n ORDER BY \n town,\n "date"\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, "date")\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n price UInt32,\n "date" Date,\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String)\n)\nAS SELECT\n price,\n "date",\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n "month" Date,\n min_price SimpleAggregateFunction("min", UInt32),\n max_price SimpleAggregateFunction("max", UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY "month"\nORDER BY "month"\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n "month" Date,\n min_price SimpleAggregateFunction("min", UInt32),\n max_price SimpleAggregateFunction("max", UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth("date") AS "month"\nSELECT\n "month",\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY "month" +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n "date" DateTime64,\n variable Decimal32(2),\n fixed Decimal32(2),\n bank Decimal32(2)\n)\nPRIMARY KEY "date"\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: when_necessary & DoubleQuotes +CREATE TABLE default.uk_price_paid\n(\n "Table" String,\n "Engine" String,\n "price" UInt32,\n "date" Date,\n "postcode1" LowCardinality(String),\n "postcode2" LowCardinality(String),\n "type" Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n "is_new" UInt8,\n "duration" Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "locality" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n "price" UInt32,\n "date" Date,\n "addr1" String,\n "addr2" String,\n "street" LowCardinality(String),\n "town" LowCardinality(String),\n "district" LowCardinality(String),\n "county" LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction(min, UInt32),\n "max_price" SimpleAggregateFunction(max, UInt32),\n "volume" AggregateFunction(count, UInt32),\n "avg_price" AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n "month" Date,\n "min_price" SimpleAggregateFunction(min, UInt32),\n "max_price" SimpleAggregateFunction(max, UInt32),\n "volume" AggregateFunction(count, UInt32),\n "avg_price" AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n "date" DateTime64,\n "variable" Decimal32(2),\n "fixed" Decimal32(2),\n "bank" Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: always & BackticksMySQL +CREATE TABLE `default`.`uk_price_paid`\n(\n `Table` String,\n `Engine` String,\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX `county_index` `county` TYPE set(10) GRANULARITY 1,\n PROJECTION `town_date_projection`\n (\n SELECT \n `town`,\n `date`,\n `price`\n ORDER BY \n `town`,\n `date`\n ),\n PROJECTION `handy_aggs_projection`\n (\n SELECT \n avg(`price`),\n max(`price`),\n sum(`price`)\n GROUP BY `town`\n )\n)\nENGINE = MergeTree\nORDER BY (`postcode1`, `postcode2`, `date`)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW `default`.`prices_by_year_view` TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n `price`,\n `date`,\n `addr1`,\n `addr2`,\n `street`,\n `town`,\n `district`,\n `county`\nFROM `default`.`uk_price_paid` +CREATE TABLE `default`.`uk_prices_aggs_dest`\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW `default`.`uk_prices_aggs_view` TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(`min`, UInt32),\n `max_price` SimpleAggregateFunction(`max`, UInt32),\n `volume` AggregateFunction(`count`, UInt32),\n `avg_price` AggregateFunction(`avg`, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(`price`) AS `min_price`,\n maxSimpleState(`price`) AS `max_price`,\n countState(`price`) AS `volume`,\n avgState(`price`) AS `avg_price`\nFROM `default`.`uk_price_paid`\nGROUP BY `month` +CREATE DICTIONARY `default`.`uk_mortgage_rates_dict`\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: user_display & BackticksMySQL +CREATE TABLE default.uk_price_paid\n(\n `Table` String,\n `Engine` String,\n price UInt32,\n `date` Date,\n postcode1 LowCardinality(String),\n postcode2 LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n is_new UInt8,\n duration Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n locality LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n `date`,\n price\n ORDER BY \n town,\n `date`\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, `date`)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n price UInt32,\n `date` Date,\n addr1 String,\n addr2 String,\n street LowCardinality(String),\n town LowCardinality(String),\n district LowCardinality(String),\n county LowCardinality(String)\n)\nAS SELECT\n price,\n `date`,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n min_price SimpleAggregateFunction(`min`, UInt32),\n max_price SimpleAggregateFunction(`max`, UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `month`\nORDER BY `month`\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n min_price SimpleAggregateFunction(`min`, UInt32),\n max_price SimpleAggregateFunction(`max`, UInt32),\n volume AggregateFunction(count, UInt32),\n avg_price AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(`date`) AS `month`\nSELECT\n `month`,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY `month` +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n variable Decimal32(2),\n fixed Decimal32(2),\n bank Decimal32(2)\n)\nPRIMARY KEY `date`\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) +Settings: when_necessary & BackticksMySQL +CREATE TABLE default.uk_price_paid\n(\n `Table` String,\n `Engine` String,\n `price` UInt32,\n `date` Date,\n `postcode1` LowCardinality(String),\n `postcode2` LowCardinality(String),\n `type` Enum8(\'other\' = 0, \'terraced\' = 1, \'semi-detached\' = 2, \'detached\' = 3, \'flat\' = 4),\n `is_new` UInt8,\n `duration` Enum8(\'unknown\' = 0, \'freehold\' = 1, \'leasehold\' = 2),\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `locality` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String),\n INDEX county_index county TYPE set(10) GRANULARITY 1,\n PROJECTION town_date_projection\n (\n SELECT \n town,\n date,\n price\n ORDER BY \n town,\n date\n ),\n PROJECTION handy_aggs_projection\n (\n SELECT \n avg(price),\n max(price),\n sum(price)\n GROUP BY town\n )\n)\nENGINE = MergeTree\nORDER BY (postcode1, postcode2, date)\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.prices_by_year_view TO default.prices_by_year_dest\n(\n `price` UInt32,\n `date` Date,\n `addr1` String,\n `addr2` String,\n `street` LowCardinality(String),\n `town` LowCardinality(String),\n `district` LowCardinality(String),\n `county` LowCardinality(String)\n)\nAS SELECT\n price,\n date,\n addr1,\n addr2,\n street,\n town,\n district,\n county\nFROM default.uk_price_paid +CREATE TABLE default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY month\nORDER BY month\nSETTINGS index_granularity = 8192 +CREATE MATERIALIZED VIEW default.uk_prices_aggs_view TO default.uk_prices_aggs_dest\n(\n `month` Date,\n `min_price` SimpleAggregateFunction(min, UInt32),\n `max_price` SimpleAggregateFunction(max, UInt32),\n `volume` AggregateFunction(count, UInt32),\n `avg_price` AggregateFunction(avg, UInt32)\n)\nAS WITH toStartOfMonth(date) AS month\nSELECT\n month,\n minSimpleState(price) AS min_price,\n maxSimpleState(price) AS max_price,\n countState(price) AS volume,\n avgState(price) AS avg_price\nFROM default.uk_price_paid\nGROUP BY month +CREATE DICTIONARY default.uk_mortgage_rates_dict\n(\n `date` DateTime64,\n `variable` Decimal32(2),\n `fixed` Decimal32(2),\n `bank` Decimal32(2)\n)\nPRIMARY KEY date\nSOURCE(CLICKHOUSE(TABLE \'uk_mortgage_rates\'))\nLIFETIME(MIN 0 MAX 2628000000)\nLAYOUT(COMPLEX_KEY_HASHED()) diff --git a/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.sql b/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.sql new file mode 100644 index 00000000000..044b3a41f48 --- /dev/null +++ b/tests/queries/0_stateless/03230_show_create_query_identifier_quoting_style.sql @@ -0,0 +1,388 @@ +DROP DICTIONARY IF EXISTS uk_mortgage_rates_dict; +DROP TABLE IF EXISTS uk_mortgage_rates; +DROP VIEW IF EXISTS uk_prices_aggs_view; +DROP TABLE IF EXISTS uk_prices_aggs_dest; +DROP VIEW IF EXISTS prices_by_year_view; +DROP TABLE IF EXISTS prices_by_year_dest; +DROP TABLE IF EXISTS uk_price_paid; + +-- Create tables, views, dictionaries + +CREATE TABLE uk_price_paid +( + Table String, + Engine String, + price UInt32, + date Date, + postcode1 LowCardinality(String), + postcode2 LowCardinality(String), + type Enum('terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4, 'other' = 0), + is_new UInt8, + duration Enum('freehold' = 1, 'leasehold' = 2, 'unknown' = 0), + addr1 String, + addr2 String, + street LowCardinality(String), + locality LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String), + INDEX county_index county TYPE set(10) GRANULARITY 1, + PROJECTION town_date_projection + ( + SELECT + town, + date, + price + ORDER BY + town, + date + ), + PROJECTION handy_aggs_projection + ( + SELECT + avg(price), + max(price), + sum(price) + GROUP BY town + ) +) +ENGINE = MergeTree +ORDER BY (postcode1, postcode2, date); + +CREATE TABLE prices_by_year_dest ( + Table String, + Engine String, + price UInt32, + date Date, + addr1 String, + addr2 String, + street LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String) +) +ENGINE = MergeTree +PRIMARY KEY (town, date) +PARTITION BY toYear(date); + +CREATE MATERIALIZED VIEW prices_by_year_view +TO prices_by_year_dest +AS + SELECT + price, + date, + addr1, + addr2, + street, + town, + district, + county + FROM uk_price_paid; + +CREATE TABLE uk_prices_aggs_dest ( + month Date, + min_price SimpleAggregateFunction(min, UInt32), + max_price SimpleAggregateFunction(max, UInt32), + volume AggregateFunction(count, UInt32), + avg_price AggregateFunction(avg, UInt32) +) +ENGINE = AggregatingMergeTree +PRIMARY KEY month; + +CREATE MATERIALIZED VIEW uk_prices_aggs_view +TO uk_prices_aggs_dest +AS + WITH + toStartOfMonth(date) AS month + SELECT + month, + minSimpleState(price) AS min_price, + maxSimpleState(price) AS max_price, + countState(price) AS volume, + avgState(price) AS avg_price + FROM uk_price_paid + GROUP BY month; + +CREATE TABLE uk_mortgage_rates ( + date DateTime64, + variable Decimal32(2), + fixed Decimal32(2), + bank Decimal32(2) +) +ENGINE Memory(); + +INSERT INTO uk_mortgage_rates VALUES ('2004-02-29', 5.02, 4.9, 4); +INSERT INTO uk_mortgage_rates VALUES ('2004-03-31', 5.11, 4.91, 4); + +CREATE DICTIONARY uk_mortgage_rates_dict ( + date DateTime64, + variable Decimal32(2), + fixed Decimal32(2), + bank Decimal32(2) +) +PRIMARY KEY date +SOURCE( + CLICKHOUSE(TABLE 'uk_mortgage_rates') +) +LAYOUT(COMPLEX_KEY_HASHED()) +LIFETIME(2628000000); + + +-- Show tables, views, dictionaries with default settings +SELECT('Settings: default'); +SHOW CREATE TABLE uk_price_paid; +SHOW CREATE VIEW prices_by_year_view; +SHOW CREATE uk_prices_aggs_dest; +SHOW CREATE VIEW uk_prices_aggs_view; +SHOW CREATE DICTIONARY uk_mortgage_rates_dict; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='always', show_create_query_identifier_quoting_style='Backticks' +SELECT('Settings: always & Backticks'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='Backticks'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='user_display', show_create_query_identifier_quoting_style='Backticks' +SELECT('Settings: user_display & Backticks'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='Backticks'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='when_necessary', show_create_query_identifier_quoting_style='Backticks' +SELECT('Settings: when_necessary & Backticks'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='Backticks'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='Backticks'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='always', show_create_query_identifier_quoting_style='DoubleQuotes' +SELECT('Settings: always & DoubleQuotes'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='user_display', show_create_query_identifier_quoting_style='DoubleQuotes' +SELECT('Settings: user_display & DoubleQuotes'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='when_necessary', show_create_query_identifier_quoting_style='DoubleQuotes' +SELECT('Settings: when_necessary & DoubleQuotes'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='DoubleQuotes'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='always', show_create_query_identifier_quoting_style='BackticksMySQL' +SELECT('Settings: always & BackticksMySQL'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='always', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='user_display', show_create_query_identifier_quoting_style='BackticksMySQL' +SELECT('Settings: user_display & BackticksMySQL'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='user_display', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +-- Show tables, views, dictionaries with show_create_query_identifier_quoting_rule='when_necessary', show_create_query_identifier_quoting_style='BackticksMySQL' +SELECT('Settings: when_necessary & BackticksMySQL'); +SHOW CREATE TABLE uk_price_paid +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW prices_by_year_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE uk_prices_aggs_dest +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE VIEW uk_prices_aggs_view +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +SHOW CREATE DICTIONARY uk_mortgage_rates_dict +SETTINGS + show_create_query_identifier_quoting_rule='when_necessary', + show_create_query_identifier_quoting_style='BackticksMySQL'; + +DROP DICTIONARY uk_mortgage_rates_dict; +DROP TABLE uk_mortgage_rates; +DROP VIEW uk_prices_aggs_view; +DROP TABLE uk_prices_aggs_dest; +DROP VIEW prices_by_year_view; +DROP TABLE prices_by_year_dest; +DROP TABLE uk_price_paid; diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.reference b/tests/queries/0_stateless/03231_create_with_clone_as.reference new file mode 100644 index 00000000000..d2046e19a23 --- /dev/null +++ b/tests/queries/0_stateless/03231_create_with_clone_as.reference @@ -0,0 +1,42 @@ +CREATE TABLE default.foo_memory\n(\n `x` Int8,\n `y` String\n)\nENGINE = Memory +CREATE TABLE default.foo_file\n(\n `x` Int8,\n `y` String\n)\nENGINE = File(\'TabSeparated\') +CREATE TABLE default.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +CREATE TABLE default.clone_as_foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_merge_tree +1 a +2 b +from clone_as_foo_merge_tree +1 a +2 b +from clone_as_foo_merge_tree_p_x +1 a +2 b +CREATE TABLE default.foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +1 a +2 b +CREATE TABLE default.clone_as_foo_replacing_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplacingMergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_replacing_merge_tree +1 a +2 b +from clone_as_foo_replacing_merge_tree +1 a +2 b +from clone_as_foo_replacing_merge_tree_p_x +1 a +2 b +CREATE TABLE default.foo_replicated_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/test_foo_replicated_merge_tree\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_replicated_merge_tree +1 a +2 b +CREATE TABLE default.clone_as_foo_replicated_merge_tree_p_x\n(\n `x` Int8,\n `y` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/clone_as_foo_replicated_merge_tree_p_x\', \'r1\')\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from clone_as_foo_replicated_merge_tree_p_x +1 a +2 b +s1 r1 OK 0 0 +CREATE TABLE default_1.foo_merge_tree\n(\n `x` Int8,\n `y` String\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +from foo_merge_tree +1 a +2 b +s1 r1 OK 0 0 diff --git a/tests/queries/0_stateless/03231_create_with_clone_as.sql b/tests/queries/0_stateless/03231_create_with_clone_as.sql new file mode 100644 index 00000000000..c5793206e88 --- /dev/null +++ b/tests/queries/0_stateless/03231_create_with_clone_as.sql @@ -0,0 +1,121 @@ +-- Tags: no-replicated-database +-- Tag no-replicated-database: Unsupported type of CREATE TABLE ... CLONE AS ... query + +DROP TABLE IF EXISTS foo_memory; +DROP TABLE IF EXISTS clone_as_foo_memory; +DROP TABLE IF EXISTS foo_file; +DROP TABLE IF EXISTS clone_as_foo_file; +DROP TABLE IF EXISTS foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_y; +DROP TABLE IF EXISTS foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_y; +DROP TABLE IF EXISTS foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_y; + +-- CLONE AS with a table of Memory engine +CREATE TABLE foo_memory (x Int8, y String) ENGINE=Memory; +SHOW CREATE TABLE foo_memory; +INSERT INTO foo_memory VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE clone_as_foo_memory CLONE AS foo_memory; -- { serverError SUPPORT_IS_DISABLED } + +-- CLONE AS with a table of File engine +CREATE TABLE foo_file (x Int8, y String) ENGINE=File(TabSeparated); +SHOW CREATE TABLE foo_file; +INSERT INTO foo_file VALUES (1, 'a'), (2, 'b'); + +CREATE TABLE clone_as_foo_file CLONE AS foo_file; -- { serverError SUPPORT_IS_DISABLED } + +-- CLONE AS with a table of MergeTree engine +CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_merge_tree; +INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT * FROM foo_merge_tree; + +CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree; +SHOW CREATE TABLE clone_as_foo_merge_tree; +SELECT 'from foo_merge_tree'; +SELECT * FROM foo_merge_tree; +SELECT 'from clone_as_foo_merge_tree'; +SELECT * FROM clone_as_foo_merge_tree; + +-- Specify ENGINE +CREATE TABLE clone_as_foo_merge_tree_p_x CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY x; +SELECT 'from clone_as_foo_merge_tree_p_x'; +SELECT * FROM clone_as_foo_merge_tree_p_x; +CREATE TABLE clone_as_foo_merge_tree_p_y CLONE AS foo_merge_tree ENGINE=MergeTree PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } + +-- CLONE AS with a table of ReplacingMergeTree engine +CREATE TABLE foo_replacing_merge_tree (x Int8, y String) ENGINE=ReplacingMergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_replacing_merge_tree; +INSERT INTO foo_replacing_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT * FROM foo_replacing_merge_tree; + +CREATE TABLE clone_as_foo_replacing_merge_tree CLONE AS foo_replacing_merge_tree; +SHOW CREATE TABLE clone_as_foo_replacing_merge_tree; +SELECT 'from foo_replacing_merge_tree'; +SELECT * FROM foo_replacing_merge_tree; +SELECT 'from clone_as_foo_replacing_merge_tree'; +SELECT * FROM clone_as_foo_replacing_merge_tree; + +-- Specify ENGINE +CREATE TABLE clone_as_foo_replacing_merge_tree_p_x CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY x; +SELECT 'from clone_as_foo_replacing_merge_tree_p_x'; +SELECT * FROM clone_as_foo_replacing_merge_tree_p_x; +CREATE TABLE clone_as_foo_replacing_merge_tree_p_y CLONE AS foo_replacing_merge_tree ENGINE=ReplacingMergeTree PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } + +-- CLONE AS with a table of ReplicatedMergeTree engine +CREATE TABLE foo_replicated_merge_tree (x Int8, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_foo_replicated_merge_tree', 'r1') PRIMARY KEY x; +SHOW CREATE TABLE foo_replicated_merge_tree; +INSERT INTO foo_replicated_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT 'from foo_replicated_merge_tree'; +SELECT * FROM foo_replicated_merge_tree; + +CREATE TABLE clone_as_foo_replicated_merge_tree CLONE AS foo_replicated_merge_tree; -- { serverError REPLICA_ALREADY_EXISTS } + +-- Specify ENGINE +CREATE TABLE clone_as_foo_replicated_merge_tree_p_x CLONE AS foo_replicated_merge_tree ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/clone_as_foo_replicated_merge_tree_p_x', 'r1') PRIMARY KEY x; +SHOW CREATE TABLE clone_as_foo_replicated_merge_tree_p_x; +SELECT 'from clone_as_foo_replicated_merge_tree_p_x'; +SELECT * FROM foo_replicated_merge_tree; +CREATE TABLE clone_as_foo_replicated_merge_tree_p_y CLONE AS foo_replicated_merge_tree ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/clone_as_foo_replicated_merge_tree_p_y', 'r1') PRIMARY KEY y; -- { serverError BAD_ARGUMENTS } + +DROP TABLE IF EXISTS foo_memory; +DROP TABLE IF EXISTS clone_as_foo_memory; +DROP TABLE IF EXISTS foo_file; +DROP TABLE IF EXISTS clone_as_foo_file; +DROP TABLE IF EXISTS foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_merge_tree_p_y; +DROP TABLE IF EXISTS foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replacing_merge_tree_p_y; +DROP TABLE IF EXISTS foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_x; +DROP TABLE IF EXISTS clone_as_foo_replicated_merge_tree_p_y; + +-- CLONE AS with a Replicated database +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Replicated('/test/databases/{database}/test_03231', 's1', 'r1'); +USE {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE TABLE foo_merge_tree (x Int8, y String) ENGINE=MergeTree PRIMARY KEY x; +SHOW CREATE TABLE foo_merge_tree; +INSERT INTO foo_merge_tree VALUES (1, 'a'), (2, 'b'); +SELECT 'from foo_merge_tree'; +SELECT * FROM foo_merge_tree; +CREATE TABLE clone_as_foo_merge_tree CLONE AS foo_merge_tree; -- { serverError SUPPORT_IS_DISABLED } + +DROP TABLE IF EXISTS clone_as_foo_merge_tree; +DROP TABLE IF EXISTS foo_merge_tree; +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; \ No newline at end of file diff --git a/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.reference b/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.reference new file mode 100644 index 00000000000..8268df7a65b --- /dev/null +++ b/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.reference @@ -0,0 +1,7 @@ +CREATE USER user_03231 IDENTIFIED WITH no_password DEFAULT ROLE role_a_03231 SETTINGS custom_x = \'x\' +GRANT role_a_03231 TO user_03231 +CREATE ROLE role_a_03231 +GRANT INSERT ON *.* TO role_a_03231 +GRANT role_b_03231 TO role_a_03231 +CREATE ROLE role_b_03231 +GRANT SELECT ON *.* TO role_b_03231 diff --git a/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.sh b/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.sh new file mode 100755 index 00000000000..550db582b30 --- /dev/null +++ b/tests/queries/0_stateless/03231_old_backup_without_access_entities_dependents.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# Tag no-fasttest: we restore from a zip-archived backup here. +# Tag no-parallel: we drop and restore fixed users and roles. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# In this test we restore from "/tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip" +backup_name="$($CURDIR/helpers/install_predefined_backup.sh old_backup_without_access_entities_dependents.zip)" + +${CLICKHOUSE_CLIENT} -m --query " +DROP USER IF EXISTS user_03231; +DROP ROLE IF EXISTS role_a_03231, role_b_03231; +" + +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM Disk('backups', '${backup_name}') FORMAT Null" + +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER user_03231" +${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR user_03231" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE ROLE role_a_03231" +${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR role_a_03231" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE ROLE role_b_03231" +${CLICKHOUSE_CLIENT} --query "SHOW GRANTS FOR role_b_03231" + +${CLICKHOUSE_CLIENT} -m --query " +DROP USER user_03231; +DROP ROLE role_a_03231, role_b_03231; +" diff --git a/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference b/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference index cad1bf13574..70ff4078811 100644 --- a/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference +++ b/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference @@ -1,6 +1,13 @@ Everything dropped User dropped +Role dropped Nothing dropped Nothing dropped, mode=replace Nothing dropped, mode=create ACCESS_ENTITY_ALREADY_EXISTS +Everything dropped, restore system.roles, then system.users +user_a 0 +role_b 1 +Everything dropped, restore system.users, then system.roles +user_a 1 +role_b 0 diff --git a/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh b/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh index 04f907b719d..e24ce4a33f5 100755 --- a/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh +++ b/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh @@ -55,14 +55,10 @@ ${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}" ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" do_check -# TODO: Cannot restore a dropped role granted to an existing user. The result after RESTORE ALL below is the following: -# CREATE USER user_a DEFAULT ROLE NONE SETTINGS custom_x = 2; GRANT NONE TO user_a; CREATE ROLE role_b SETTINGS custom_x = 1 -# because `role_b` is restored but not granted to existing user `user_a`. -# -# echo "Role dropped" -# ${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" -# ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" -# do_check +echo "Role dropped" +${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" +do_check echo "Nothing dropped" ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" @@ -75,3 +71,23 @@ do_check echo "Nothing dropped, mode=create" ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} SETTINGS create_access='create' FORMAT Null" 2>&1 | grep -om1 "ACCESS_ENTITY_ALREADY_EXISTS" do_check + +echo "Everything dropped, restore system.roles, then system.users" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}" +${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" +# Here "skip_unresolved_access_dependencies=true" because users don't exist yet and restored roles can't be granted to non-existent users. +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.roles FROM ${backup_name} SETTINGS skip_unresolved_access_dependencies=true FORMAT Null" +${CLICKHOUSE_CLIENT} --query "SELECT 'user_a', count() FROM system.users WHERE name = '${user_a}'" +${CLICKHOUSE_CLIENT} --query "SELECT 'role_b', count() FROM system.roles WHERE name = '${role_b}'" +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.users FROM ${backup_name} FORMAT Null" +do_check + +echo "Everything dropped, restore system.users, then system.roles" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}" +${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" +# Here "skip_unresolved_access_dependencies=true" because roles don't exist yet and can't be granted to restored users. +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.users FROM ${backup_name} SETTINGS skip_unresolved_access_dependencies=true FORMAT Null" +${CLICKHOUSE_CLIENT} --query "SELECT 'user_a', count() FROM system.users WHERE name = '${user_a}'" +${CLICKHOUSE_CLIENT} --query "SELECT 'role_b', count() FROM system.roles WHERE name = '${role_b}'" +${CLICKHOUSE_CLIENT} --query "RESTORE TABLE system.roles FROM ${backup_name} FORMAT Null" +do_check diff --git a/tests/queries/0_stateless/03234_get_setting_or_default.reference b/tests/queries/0_stateless/03234_get_setting_or_default.reference new file mode 100644 index 00000000000..0b47065a07b --- /dev/null +++ b/tests/queries/0_stateless/03234_get_setting_or_default.reference @@ -0,0 +1,10 @@ +value_a +value_b +\N +5 +default_e +500 +\N +1 +1 +backup diff --git a/tests/queries/0_stateless/03234_get_setting_or_default.sql b/tests/queries/0_stateless/03234_get_setting_or_default.sql new file mode 100644 index 00000000000..3954e9fe8ab --- /dev/null +++ b/tests/queries/0_stateless/03234_get_setting_or_default.sql @@ -0,0 +1,24 @@ +SET custom_a = 'value_a'; +SET custom_b = 'value_b'; +SET custom_c = null; +SET custom_d = 5; + +SELECT getSettingOrDefault('custom_a', 'default_a'); +SELECT getSettingOrDefault('custom_b', 'default_b'); +SELECT getSettingOrDefault('custom_c', 'default_c'); +SELECT getSettingOrDefault('custom_d', 'default_d'); + +SELECT getSetting('custom_e'); -- { serverError UNKNOWN_SETTING } + +SELECT getSettingOrDefault('custom_e', 'default_e'); +SELECT getSettingOrDefault('custom_e', 500); +SELECT getSettingOrDefault('custom_e', null); +SELECT isNull(getSettingOrDefault('custom_e', null)); + +SELECT getSettingOrDefault('custom_e'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT getSettingOrDefault(115, 'name should be string'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +SELECT count(*) FROM numbers(10) WHERE number = getSettingOrDefault('custom_e', 5); + +SET custom_e_backup = 'backup'; +SELECT getSettingOrDefault('custom_e', getSetting('custom_e_backup')); diff --git a/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.reference b/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.reference new file mode 100644 index 00000000000..cae461492b5 --- /dev/null +++ b/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.reference @@ -0,0 +1,3 @@ +{"error_not_raised":1} +{"i":1001,"j.k":[2101,2102],"j.l":[2201,2202],"m":3001,"n":[4001,4002,4003,4004],"o.key":[5001,5002],"o.value":[{"p":[{"q":5111,"r":5121}]},{"p":[{"q":5112,"r":5122},{"q":5113,"r":5123}]}]} +{"i":6001,"j.k":[7101],"j.l":[7201],"m":8001,"n":[],"o.key":[9001],"o.value":[{"p":[{"q":10111,"r":10121}]}]} diff --git a/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.sh b/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.sh new file mode 100755 index 00000000000..47541dd4f5d --- /dev/null +++ b/tests/queries/0_stateless/03234_proto_complex_nested_repeated_noexception.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +mkdir -p "${CLICKHOUSE_SCHEMA_FILES}" +SOURCE_SCHEMA_FILE="${CURDIR}/format_schemas/03234_proto_complex_nested_repeated_noexception.proto" +TARGET_SCHEMA_FILE="${CLICKHOUSE_SCHEMA_FILES}/03234_proto_complex_nested_repeated_noexception.proto" +cp "${SOURCE_SCHEMA_FILE}" "${TARGET_SCHEMA_FILE}" + +cat <<'EOF' | $CLICKHOUSE_CLIENT -mn + +DROP TABLE IF EXISTS exception_counter ; +CREATE TABLE exception_counter (`val` UInt32) ENGINE = Memory ; +INSERT INTO exception_counter SELECT sum(value) FROM system.errors WHERE name = 'PROTOBUF_FIELD_NOT_REPEATED' ; + +DROP TABLE IF EXISTS table_file ; +CREATE TABLE table_file ( + `i` UInt32, + `j.k` Array(UInt32), + `j.l` Array(UInt32), + `m` UInt32, + `n` Array(UInt32), + `o` Nested( + `key` UInt32, + `value` Tuple( + `p` Nested( + `q` UInt32, + `r` UInt32 + ) + ) + ) +) ENGINE File(Protobuf) SETTINGS format_schema = '03234_proto_complex_nested_repeated_noexception.proto:A' ; + +INSERT INTO table_file VALUES +( 1001, [2101, 2102], [2201, 2202], 3001, [4001, 4002, 4003, 4004], [5001,5002] , [ ([(5111,5121)]), ([(5112,5122),(5113,5123)]) ] ), +( 6001, [7101], [7201], 8001, [], [9001] , [ ([(10111,10121)]) ] ) ; + +INSERT INTO exception_counter SELECT sum(value) FROM system.errors WHERE name = 'PROTOBUF_FIELD_NOT_REPEATED' ; +SELECT min(val) == max(val) as error_not_raised FROM exception_counter FORMAT JSONEachRow ; + +SELECT * FROM table_file FORMAT JSONEachRow ; + +DROP TABLE exception_counter ; +DROP TABLE table_file ; + +EOF + +rm -f "${TARGET_SCHEMA_FILE}" diff --git a/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.reference b/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.reference new file mode 100644 index 00000000000..8618ac3e072 --- /dev/null +++ b/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.reference @@ -0,0 +1,5 @@ +{"error_not_raised":1} +{"u":1001,"v.w":[],"v.x":[],"v.y":[],"v.z":[]} +{"u":2002,"v.w":[2102],"v.x":[2202],"v.y":[[2302]],"v.z":[[2402,2403]]} +{"u":3003,"v.w":[3103,3104],"v.x":[3203,3204],"v.y":[[3303],[3304]],"v.z":[[3403,3404],[3405,3406]]} +{"u":4004,"v.w":[4104,4105,4106],"v.x":[4204,4205,4206],"v.y":[[4304],[4305],[4306]],"v.z":[[4304,4305],[4306,4307],[4308,4309]]} diff --git a/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.sh b/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.sh new file mode 100755 index 00000000000..a20c2ba12a6 --- /dev/null +++ b/tests/queries/0_stateless/03234_proto_simple_nested_repeated_noexception.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +mkdir -p "${CLICKHOUSE_SCHEMA_FILES}" +SOURCE_SCHEMA_FILE="${CURDIR}/format_schemas/03234_proto_simple_nested_repeated_noexception.proto" +TARGET_SCHEMA_FILE="${CLICKHOUSE_SCHEMA_FILES}/03234_proto_simple_nested_repeated_noexception.proto" +cp "${SOURCE_SCHEMA_FILE}" "${TARGET_SCHEMA_FILE}" + +cat <<'EOF' | $CLICKHOUSE_CLIENT -mn + +DROP TABLE IF EXISTS exception_counter ; +CREATE TABLE exception_counter (`val` UInt32) ENGINE = Memory ; +INSERT INTO exception_counter SELECT sum(value) FROM system.errors WHERE name = 'PROTOBUF_FIELD_NOT_REPEATED' ; + +DROP TABLE IF EXISTS table_file ; +CREATE TABLE table_file ( + `u` UInt32, + `v.w` Array(UInt32), + `v.x` Array(UInt32), + `v.y` Array(Array(UInt32)), + `v.z` Array(Array(UInt32)) +) ENGINE File(Protobuf) SETTINGS format_schema = '03234_proto_simple_nested_repeated_noexception.proto:M' ; + +INSERT INTO table_file VALUES +( 1001, [], [], [], []), +( 2002, [2102], [2202], [[2302]], [[2402, 2403]]), +( 3003, [3103, 3104], [3203, 3204], [[3303], [3304]], [[3403, 3404], [3405, 3406]]), +( 4004, [4104, 4105, 4106], [4204, 4205, 4206], [[4304], [4305], [4306]], [[4304, 4305], [4306, 4307], [4308, 4309]]); + + +INSERT INTO exception_counter SELECT sum(value) FROM system.errors WHERE name = 'PROTOBUF_FIELD_NOT_REPEATED' ; +SELECT min(val) == max(val) as error_not_raised FROM exception_counter FORMAT JSONEachRow ; + +SELECT * FROM table_file FORMAT JSONEachRow ; + +DROP TABLE exception_counter ; +DROP TABLE table_file ; + +EOF + +rm -f "${TARGET_SCHEMA_FILE}" diff --git a/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.reference b/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.sh b/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.sh new file mode 100755 index 00000000000..cc0e6c0d113 --- /dev/null +++ b/tests/queries/0_stateless/03237_create_or_replace_view_atomically_with_atomic_engine.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# with Atomic engine +$CLICKHOUSE_CLIENT --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE}_db ENGINE=Atomic" + +function create_or_replace_view_thread +{ + for _ in {1..20}; do + $CLICKHOUSE_CLIENT --query "CREATE OR REPLACE VIEW ${CLICKHOUSE_DATABASE}_db.test_view AS SELECT 'abcdef'" > /dev/null + done +} +export -f create_or_replace_view_thread; + +function select_view_thread +{ + for _ in {1..20}; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM ${CLICKHOUSE_DATABASE}_db.test_view" > /dev/null + done +} +export -f select_view_thread; + +$CLICKHOUSE_CLIENT --query "CREATE OR REPLACE VIEW ${CLICKHOUSE_DATABASE}_db.test_view AS SELECT 'abcdef'" > /dev/null + +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & + +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & + +wait \ No newline at end of file diff --git a/tests/queries/0_stateless/03237_insert_sparse_columns.reference b/tests/queries/0_stateless/03237_insert_sparse_columns.reference new file mode 100644 index 00000000000..592fcff9b25 --- /dev/null +++ b/tests/queries/0_stateless/03237_insert_sparse_columns.reference @@ -0,0 +1,21 @@ +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 +10 0 +11 100 +12 200 +13 300 +14 400 +15 500 +all_1_1_0 id Default +all_1_1_0 v Sparse +all_2_2_0 id Default +all_2_2_0 v Sparse +all_3_3_0 id Default +all_3_3_0 v Default diff --git a/tests/queries/0_stateless/03237_insert_sparse_columns.sh b/tests/queries/0_stateless/03237_insert_sparse_columns.sh new file mode 100755 index 00000000000..a4d53a36b87 --- /dev/null +++ b/tests/queries/0_stateless/03237_insert_sparse_columns.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_insert_sparse_columns; + CREATE TABLE t_insert_sparse_columns (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5; + SYSTEM STOP MERGES t_insert_sparse_columns; +" + +${CLICKHOUSE_CURL} -sS ${CLICKHOUSE_URL} -d 'INSERT INTO t_insert_sparse_columns FORMAT JSONEachRow {"id": 1} {"id": 2} {"id": 3} {"id": 4} {"id": 5}' +${CLICKHOUSE_CURL} -sS ${CLICKHOUSE_URL} -d 'INSERT INTO t_insert_sparse_columns FORMAT JSONEachRow {"id": 6} {"id": 7} {"id": 8} {"id": 9} {"id": 10}' +${CLICKHOUSE_CURL} -sS ${CLICKHOUSE_URL} -d 'INSERT INTO t_insert_sparse_columns FORMAT JSONEachRow {"id": 11, "v": 100} {"id": 12, "v": 200} {"id": 13, "v": 300} {"id": 14, "v": 400} {"id": 15, "v": 500}' + +$CLICKHOUSE_CLIENT --query " + SELECT * FROM t_insert_sparse_columns ORDER BY id; + + SELECT name, column, serialization_kind FROM system.parts_columns + WHERE table = 't_insert_sparse_columns' AND database = currentDatabase() AND active + ORDER BY name, column; + + DROP TABLE t_insert_sparse_columns; +" diff --git a/tests/queries/0_stateless/03237_insert_sparse_columns_mem.reference b/tests/queries/0_stateless/03237_insert_sparse_columns_mem.reference new file mode 100644 index 00000000000..09ef3399bad --- /dev/null +++ b/tests/queries/0_stateless/03237_insert_sparse_columns_mem.reference @@ -0,0 +1,9 @@ +120000 +435170936075214220 +435170936075214220 +Default 4 +Sparse 1000 +0 +1 +1 +1 diff --git a/tests/queries/0_stateless/03237_insert_sparse_columns_mem.sh b/tests/queries/0_stateless/03237_insert_sparse_columns_mem.sh new file mode 100755 index 00000000000..ac682a0f574 --- /dev/null +++ b/tests/queries/0_stateless/03237_insert_sparse_columns_mem.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +table_structure="id UInt64" + +for i in {1..250}; do + table_structure+=", c$i String" +done + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_insert_mem; + DROP TABLE IF EXISTS t_reference; + + CREATE TABLE t_insert_mem ($table_structure) ENGINE = MergeTree ORDER BY id SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + CREATE TABLE t_reference ($table_structure) ENGINE = Log; + + SYSTEM STOP MERGES t_insert_mem; +" + +filename="test_data_sparse_$CLICKHOUSE_DATABASE.json" + +$CLICKHOUSE_CLIENT --query " + INSERT INTO FUNCTION file('$filename', LineAsString) + SELECT format('{{ \"id\": {}, \"c{}\": \"{}\" }}', number, number % 250, hex(number * 1000000)) FROM numbers(30000) + SETTINGS engine_file_truncate_on_insert = 1; + + INSERT INTO FUNCTION s3(s3_conn, filename='$filename', format='LineAsString') + SELECT * FROM file('$filename', LineAsString) + SETTINGS s3_truncate_on_insert = 1; +" + +for _ in {1..4}; do + $CLICKHOUSE_CLIENT --query "INSERT INTO t_reference SELECT * FROM file('$filename', JSONEachRow)" +done; + +$CLICKHOUSE_CLIENT --enable_parsing_to_custom_serialization 1 --query "INSERT INTO t_insert_mem SELECT * FROM file('$filename', JSONEachRow)" +$CLICKHOUSE_CLIENT --enable_parsing_to_custom_serialization 1 --query "INSERT INTO t_insert_mem SELECT * FROM file('$filename', JSONEachRow)" +$CLICKHOUSE_CLIENT --enable_parsing_to_custom_serialization 1 --query "INSERT INTO t_insert_mem SELECT * FROM s3(s3_conn, filename='$filename', format='JSONEachRow')" +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('$filename', LineAsString) FORMAT LineAsString" | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_insert_mem+FORMAT+JSONEachRow&enable_parsing_to_custom_serialization=1" --data-binary @- + +$CLICKHOUSE_CLIENT --query " + SELECT count() FROM t_insert_mem; + SELECT sum(sipHash64(*)) FROM t_insert_mem; + SELECT sum(sipHash64(*)) FROM t_reference; + + SELECT serialization_kind, count() FROM system.parts_columns + WHERE table = 't_insert_mem' AND database = '$CLICKHOUSE_DATABASE' + GROUP BY serialization_kind ORDER BY serialization_kind; + + SYSTEM FLUSH LOGS; + + SELECT written_bytes <= 3000000 FROM system.query_log + WHERE query LIKE 'INSERT INTO t_insert_mem%' AND current_database = '$CLICKHOUSE_DATABASE' AND type = 'QueryFinish' + ORDER BY event_time_microseconds; + + DROP TABLE IF EXISTS t_insert_mem; + DROP TABLE IF EXISTS t_reference; +" diff --git a/tests/queries/0_stateless/03238_create_or_replace_view_atomically_with_replicated_engine.reference b/tests/queries/0_stateless/03238_create_or_replace_view_atomically_with_replicated_engine.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03238_create_or_replace_view_atomically_with_replicated_engine.sh b/tests/queries/0_stateless/03238_create_or_replace_view_atomically_with_replicated_engine.sh new file mode 100755 index 00000000000..04adc38e34b --- /dev/null +++ b/tests/queries/0_stateless/03238_create_or_replace_view_atomically_with_replicated_engine.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# with Replicated engine +$CLICKHOUSE_CLIENT --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE}_db ENGINE=Replicated('/test/clickhouse/db/${CLICKHOUSE_DATABASE}_db', 's1', 'r1')" + +function create_or_replace_view_thread +{ + for _ in {1..15}; do + $CLICKHOUSE_CLIENT --query "CREATE OR REPLACE VIEW ${CLICKHOUSE_DATABASE}_db.test_view AS SELECT 'abcdef'" > /dev/null + done +} +export -f create_or_replace_view_thread; + +function select_view_thread +{ + for _ in {1..15}; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM ${CLICKHOUSE_DATABASE}_db.test_view" > /dev/null + done +} +export -f select_view_thread; + +$CLICKHOUSE_CLIENT --query "CREATE OR REPLACE VIEW ${CLICKHOUSE_DATABASE}_db.test_view AS SELECT 'abcdef'" > /dev/null + +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & +bash -c select_view_thread & + +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & +bash -c create_or_replace_view_thread & + +wait \ No newline at end of file diff --git a/tests/queries/0_stateless/03243_array_join_lambda.reference b/tests/queries/0_stateless/03243_array_join_lambda.reference new file mode 100644 index 00000000000..057d00e5851 --- /dev/null +++ b/tests/queries/0_stateless/03243_array_join_lambda.reference @@ -0,0 +1 @@ +[2,3] diff --git a/tests/queries/0_stateless/03243_array_join_lambda.sql b/tests/queries/0_stateless/03243_array_join_lambda.sql new file mode 100644 index 00000000000..0cd93a8d781 --- /dev/null +++ b/tests/queries/0_stateless/03243_array_join_lambda.sql @@ -0,0 +1 @@ +SELECT arrayMap(x -> (x + length(arrayJoin([arrayMap(y -> (y + 1), [3])]))), [1, 2]); diff --git a/tests/queries/0_stateless/03245_ripemd160.reference b/tests/queries/0_stateless/03245_ripemd160.reference new file mode 100644 index 00000000000..6227b2277a4 --- /dev/null +++ b/tests/queries/0_stateless/03245_ripemd160.reference @@ -0,0 +1,5 @@ +37F332F68DB77BD9D7EDD4969571AD671CF9DD3B +132072DF690933835EB8B6AD0B77E7B6F14ACAD7 +9C1185A5C5E9FC54612808977EE8F548B2258D31 +C5BE7750205377BD79C43764134E3DBC0280B524 +792C23EA284363927133CD009CF2C08937265D11 diff --git a/tests/queries/0_stateless/03245_ripemd160.sql b/tests/queries/0_stateless/03245_ripemd160.sql new file mode 100644 index 00000000000..2b1df2ffe7e --- /dev/null +++ b/tests/queries/0_stateless/03245_ripemd160.sql @@ -0,0 +1,12 @@ +-- Tags: no-fasttest + +SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy dog')); + +SELECT HEX(RIPEMD160('The quick brown fox jumps over the lazy cog')); + +SELECT HEX(RIPEMD160('')); + +SELECT HEX(RIPEMD160('A-very-long-string-that-should-be-hashed-using-ripemd160')); + +SELECT HEX(RIPEMD160(toString(avg(number))) ) +FROM (SELECT arrayJoin([1, 2, 3, 4, 5]) AS number); diff --git a/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.reference b/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.reference new file mode 100644 index 00000000000..12442cadf31 --- /dev/null +++ b/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.reference @@ -0,0 +1 @@ +1 2024-05-02 00:00:00 diff --git a/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.sql b/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.sql new file mode 100644 index 00000000000..be5f27b55de --- /dev/null +++ b/tests/queries/0_stateless/03245_views_and_filter_push_down_bug.sql @@ -0,0 +1 @@ +select * from view(select id, toDateTime(date) as date from view(select 1 as id, '2024-05-02' as date)) where date='2024-05-02'; diff --git a/tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip b/tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip new file mode 100644 index 00000000000..a20e66b2f6b Binary files /dev/null and b/tests/queries/0_stateless/backups/old_backup_without_access_entities_dependents.zip differ diff --git a/tests/queries/0_stateless/format_schemas/03234_proto_complex_nested_repeated_noexception.proto b/tests/queries/0_stateless/format_schemas/03234_proto_complex_nested_repeated_noexception.proto new file mode 100644 index 00000000000..02d88dbd719 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/03234_proto_complex_nested_repeated_noexception.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +message A { + message B { + uint32 k = 2100 ; + uint32 l = 2200 ; + } + + uint32 i = 1000 ; + repeated B j = 2000 ; + uint32 m = 3000 ; + + repeated uint32 n = 4000 ; + + map o = 5000 ; +} + +message D { + uint32 q = 5110 ; + uint32 r = 5120 ; +} + +message C { + repeated D p = 5100 ; +} diff --git a/tests/queries/0_stateless/format_schemas/03234_proto_simple_nested_repeated_noexception.proto b/tests/queries/0_stateless/format_schemas/03234_proto_simple_nested_repeated_noexception.proto new file mode 100644 index 00000000000..fe766c4df20 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/03234_proto_simple_nested_repeated_noexception.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +message M { + uint32 u = 1000 ; + repeated N v = 2000 ; + + message N { + uint32 w = 2100 ; + uint32 x = 2200 ; + repeated uint32 y = 2300 ; + repeated uint32 z = 2400 ; + } +} diff --git a/tests/queries/0_stateless/helpers/httpexpect.py b/tests/queries/0_stateless/helpers/httpexpect.py index 6147118e793..7cdb8a3ddaf 100644 --- a/tests/queries/0_stateless/helpers/httpexpect.py +++ b/tests/queries/0_stateless/helpers/httpexpect.py @@ -11,17 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import http.client import os import sys -import http.client CURDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, CURDIR) -import uexpect +from queue import Empty, Queue +from threading import Event, Thread -from threading import Thread, Event -from queue import Queue, Empty +import uexpect class IO(uexpect.IO): diff --git a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py index c9d80dd99fc..248e884b30c 100755 --- a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py +++ b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py @@ -4,8 +4,8 @@ # To do that this script has been written. import argparse -import os.path import io +import os.path import struct import subprocess import sys diff --git a/tests/queries/0_stateless/helpers/pure_http_client.py b/tests/queries/0_stateless/helpers/pure_http_client.py index c3c4109ce5b..9a947b3ad3f 100644 --- a/tests/queries/0_stateless/helpers/pure_http_client.py +++ b/tests/queries/0_stateless/helpers/pure_http_client.py @@ -1,10 +1,11 @@ -import os import io +import os +import time + +import pandas as pd import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry -import time -import pandas as pd CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") CLICKHOUSE_PORT_HTTP = os.environ.get("CLICKHOUSE_PORT_HTTP", "8123") diff --git a/tests/queries/0_stateless/helpers/uexpect.py b/tests/queries/0_stateless/helpers/uexpect.py index 2e6d8aed19e..33707019133 100644 --- a/tests/queries/0_stateless/helpers/uexpect.py +++ b/tests/queries/0_stateless/helpers/uexpect.py @@ -13,13 +13,12 @@ # limitations under the License. import os import pty -import time -import sys import re - -from threading import Thread, Event +import sys +import time +from queue import Empty, Queue from subprocess import Popen -from queue import Queue, Empty +from threading import Event, Thread class TimeoutError(Exception): diff --git a/utils/backupview/clickhouse_backupview.py b/utils/backupview/clickhouse_backupview.py index d1331e2ab49..b3122a2a993 100755 --- a/utils/backupview/clickhouse_backupview.py +++ b/utils/backupview/clickhouse_backupview.py @@ -2,15 +2,14 @@ import bisect import os.path -import xml.etree.ElementTree as ET -from urllib.parse import urlparse import shutil - +import xml.etree.ElementTree as ET import zipfile # For reading backups from zip archives +from urllib.parse import urlparse + import boto3 # For reading backups from S3 import botocore - ## Examples: ## from backupview import open_backup ## diff --git a/utils/backupview/test/test_backupview.py b/utils/backupview/test/test_backupview.py index 5c0d546cbfa..32edc0cd328 100755 --- a/utils/backupview/test/test_backupview.py +++ b/utils/backupview/test/test_backupview.py @@ -3,17 +3,17 @@ # Tests for the clickhouse_backupview utility. # Use pytest ./test_backupview.py to run. -import pytest - import os.path import sys import tempfile +import pytest + script_dir = os.path.dirname(os.path.realpath(__file__)) backupview_dir = os.path.abspath(os.path.join(script_dir, "..")) if backupview_dir not in sys.path: sys.path.append(backupview_dir) -from clickhouse_backupview import open_backup, S3, FileInfo +from clickhouse_backupview import S3, FileInfo, open_backup def calculate_num_files(dir): diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 8ec2e001a73..2260705323b 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -750,6 +750,7 @@ PointDistM PointDistRads PostHistory PostLink +PostLinks PostgreSQLConnection PostgreSQLThreads Postgres @@ -852,6 +853,7 @@ RestartReplicaThreads RestartReplicaThreadsActive RestoreThreads RestoreThreadsActive +RIPEMD RoaringBitmap RocksDB Rollup @@ -1745,6 +1747,7 @@ getMacro getOSKernelVersion getServerPort getSetting +getSettingOrDefault getSizeOfEnumType getSubcolumn getTypeSerializationStreams @@ -3048,3 +3051,4 @@ znode znodes zookeeperSessionUptime zstd +postgres diff --git a/utils/check-style/check-isort b/utils/check-style/check-isort new file mode 100755 index 00000000000..69b2c532c6b --- /dev/null +++ b/utils/check-style/check-isort @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -e + +# We check only our code, that's why we skip contrib +GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-./} +tmp=$(mktemp) + +# Find all *.py, *.python files and executable files without extension +# that are determined as python scripts by 'file' util +# in the repo except the contrib directory. +find_cmd=( + find "$GIT_ROOT" -type f -not -path "${GIT_ROOT}contrib/*" + \( + \( + -name '*.py' -or -name "*.python" -or + \( + -executable -not -name "*.*" -exec sh -c 'file {} | grep -q "Python script"' \; + \) + \) + # We skip modules generated by the protocol buffer compiler from *.proto files. + -and -not -name '*_pb2.py' -and -not -name '*_pb2_grpc.py' + \) +) + +if ! "${find_cmd[@]}" -exec isort --check --diff {} + 1>"$tmp" 2>&1; then + # Show the result only if some files need formatting + cat "$tmp" + # Apply formatting + "${find_cmd[@]}" -exec isort {} + 1>/dev/null 2>&1 + # Automatically add changed files to stage + "${find_cmd[@]}" -exec git add -u {} + 1>/dev/null 2>&1 +fi + +rm "$tmp" diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 1b488968782..2737be85a91 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -285,9 +285,6 @@ done # There shouldn't be any code snippets under GPL or LGPL find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL" -# There shouldn't be any docker containers outside docker directory -find $ROOT_PATH -not -path $ROOT_PATH'/tests/ci*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:" - # There shouldn't be any docker compose files outside docker directory find $ROOT_PATH -name '*compose*.yml' -type f -not -path $ROOT_PATH'/docker' -not -path $ROOT_PATH'/tests/integration*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' 2>/dev/null | grep -vP $EXCLUDE_DIRS | xargs --no-run-if-empty grep -l "version:" | xargs --no-run-if-empty -n1 echo "Please move docker compose to the 'docker' or 'tests' directory:" diff --git a/utils/check-style/check_py.sh b/utils/check-style/check_py.sh index 2e645d2f19a..4e70b68ce81 100755 --- a/utils/check-style/check_py.sh +++ b/utils/check-style/check_py.sh @@ -10,6 +10,12 @@ echo "Check " | ts runtime=$((`date +%s`-start)) echo "Check python formatting with black. Done. $runtime seconds." +start=`date +%s` +echo "Check " | ts +./check-isort -n |& tee /test_output/isort_output.txt +runtime=$((`date +%s`-start)) +echo "Check python formatting with isort. Done. $runtime seconds." + start=`date +%s` ./check-pylint -n |& tee /test_output/pylint_output.txt runtime=$((`date +%s`-start)) diff --git a/utils/check-style/check_reusable_workflows.py b/utils/check-style/check_reusable_workflows.py index 6fe22786650..0a3ba096e6b 100644 --- a/utils/check-style/check_reusable_workflows.py +++ b/utils/check-style/check_reusable_workflows.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Dict, Iterable, List + import yaml git_root = Path(__file__).absolute().parents[2] diff --git a/utils/check-style/process_style_check_result.py b/utils/check-style/process_style_check_result.py index 2c349114a59..32efa70b5e2 100755 --- a/utils/check-style/process_style_check_result.py +++ b/utils/check-style/process_style_check_result.py @@ -17,6 +17,7 @@ def process_result(result_folder): "shellcheck", "style", "pylint", + "isort", "black", "flake8", "mypy", diff --git a/utils/clickhouse-diagnostics/clickhouse-diagnostics b/utils/clickhouse-diagnostics/clickhouse-diagnostics index 8e23cc8d0e1..f420d3f7cbf 100755 --- a/utils/clickhouse-diagnostics/clickhouse-diagnostics +++ b/utils/clickhouse-diagnostics/clickhouse-diagnostics @@ -12,12 +12,12 @@ from datetime import datetime from typing import MutableMapping import jinja2 -from pandas import describe_option import requests import sqlparse import tenacity import xmltodict import yaml +from pandas import describe_option SELECT_VERSION = r"SELECT version()" diff --git a/utils/data-lakes-importer.py b/utils/data-lakes-importer.py index aa03f15b1c0..0a00dd2a783 100755 --- a/utils/data-lakes-importer.py +++ b/utils/data-lakes-importer.py @@ -2,6 +2,7 @@ import os import sys + import pyspark from delta import * # pip install delta-spark diff --git a/utils/grpc-client/clickhouse-grpc-client.py b/utils/grpc-client/clickhouse-grpc-client.py index e89b6282741..dbd61477cfd 100755 --- a/utils/grpc-client/clickhouse-grpc-client.py +++ b/utils/grpc-client/clickhouse-grpc-client.py @@ -20,14 +20,23 @@ STDIN_BUFFER_SIZE = 1048576 DEFAULT_ENCODING = "utf-8" +import argparse +import cmd +import os +import signal +import sys +import threading +import time +import uuid + import grpc # pip3 install grpcio -import argparse, cmd, os, signal, sys, threading, time, uuid script_dir = os.path.dirname(os.path.realpath(__file__)) pb2_dir = os.path.join(script_dir, "pb2") if pb2_dir not in sys.path: sys.path.append(pb2_dir) -import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc # Execute pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2 # Execute pb2/generate.py to generate these modules. +import clickhouse_grpc_pb2_grpc class ClickHouseGRPCError(Exception): diff --git a/utils/kafka/consume.py b/utils/kafka/consume.py index 74542baf218..9e5369acdd1 100755 --- a/utils/kafka/consume.py +++ b/utils/kafka/consume.py @@ -1,12 +1,12 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -# `pip install …` -import kafka # … kafka-python - import argparse from pprint import pprint +# `pip install …` +import kafka # … kafka-python + def main(): parser = argparse.ArgumentParser(description="Kafka Producer client") diff --git a/utils/kafka/manage.py b/utils/kafka/manage.py index 578a7df7310..e5c89cdd412 100755 --- a/utils/kafka/manage.py +++ b/utils/kafka/manage.py @@ -1,11 +1,11 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse + # `pip install …` import kafka # … kafka-python -import argparse - def main(): parser = argparse.ArgumentParser(description="Kafka Topic manager") diff --git a/utils/kafka/produce.py b/utils/kafka/produce.py index f82e56d8478..e980fd11fb8 100755 --- a/utils/kafka/produce.py +++ b/utils/kafka/produce.py @@ -1,15 +1,15 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -# `pip install …` -import kafka # … kafka-python - import argparse -from concurrent.futures import ThreadPoolExecutor import enum import multiprocessing import sys import time +from concurrent.futures import ThreadPoolExecutor + +# `pip install …` +import kafka # … kafka-python class Sync(enum.Enum): diff --git a/utils/kafka/status.py b/utils/kafka/status.py index 12ea3d23bdf..63905650268 100755 --- a/utils/kafka/status.py +++ b/utils/kafka/status.py @@ -1,11 +1,11 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse + # `pip install …` import kafka # … kafka-python -import argparse - def main(): parser = argparse.ArgumentParser( diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index bbda26498bc..1230c47008c 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -186,6 +186,9 @@ void Runner::parseHostsFromConfig(const Poco::Util::AbstractConfiguration & conf if (config.has(key + ".use_compression")) connection_info.use_compression = config.getBool(key + ".use_compression"); + + if (config.has(key + ".use_xid_64")) + connection_info.use_xid_64 = config.getBool(key + ".use_xid_64"); }; fill_connection_details("connections", default_connection_info); @@ -1258,6 +1261,7 @@ std::shared_ptr Runner::getConnection(const ConnectionI args.connection_timeout_ms = connection_info.connection_timeout_ms; args.operation_timeout_ms = connection_info.operation_timeout_ms; args.use_compression = connection_info.use_compression; + args.use_xid_64 = connection_info.use_xid_64; return std::make_shared(nodes, args, nullptr); } diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index c19a4d82898..95d72247d5e 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -84,6 +84,7 @@ private: int32_t connection_timeout_ms = Coordination::DEFAULT_CONNECTION_TIMEOUT_MS; int32_t operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; bool use_compression = false; + bool use_xid_64 = false; size_t sessions = 1; }; diff --git a/utils/keeper-overload/keeper-overload.py b/utils/keeper-overload/keeper-overload.py index 1032ea656bc..cab41269ce2 100755 --- a/utils/keeper-overload/keeper-overload.py +++ b/utils/keeper-overload/keeper-overload.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 import os -import subprocess +import random import signal +import subprocess import sys import time -import random from argparse import ArgumentParser - XML_TEMPLATE = """ diff --git a/utils/test_history/test-history b/utils/test_history/test-history index 5f031af1d3a..8e0c7c2df4f 100755 --- a/utils/test_history/test-history +++ b/utils/test_history/test-history @@ -3,13 +3,14 @@ # Note: should work with python 2 and 3 -from github import Github -import datetime -import tabulate import argparse -from termcolor import colored +import datetime import sys +import tabulate +from github import Github +from termcolor import colored + COLORMAP = { "success": colored("success", "green"), "failure": colored("failure", "red"), diff --git a/utils/zero_copy/zero_copy_schema_converter.py b/utils/zero_copy/zero_copy_schema_converter.py index 6103ac69c6e..17f3fdcaab6 100755 --- a/utils/zero_copy/zero_copy_schema_converter.py +++ b/utils/zero_copy/zero_copy_schema_converter.py @@ -2,6 +2,7 @@ import argparse import socket import uuid + from kazoo.client import KazooClient