mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-15 10:52:30 +00:00
c9e93b21c2
Before it was possible to return stale counter from
StorageMergeTree::totalRows() (that is used for
optimize_trivial_count_query) since the accounting is done in two steps:
- subtract old number of rows
<-- here the number can be zero, even though there are rows -->
- add new number of rows
This was found by CI [1] in 01615_random_one_shard_insertion test:
Here you can see that INSERT went to both tables:
<details>
2022.01.16 09:07:34.288252 [ 154369 ] {a1905be0-93da-460c-8c6f-9b5adace72a0} <Debug> DistributedBlockOutputStream: It took 0.035197041 sec. to insert 100
blocks, 2841.1479249065287 rows per second. Insertion status:
Wrote 54 blocks and 54 rows on shard 0 replica 0, localhost:9000 (average 0 ms per block, the slowest block 1 ms)
Wrote 46 blocks and 46 rows on shard 1 replica 0, localhost:9000 (average 0 ms per block, the slowest block 1 ms)
</details>
But the test fails, since select from shard1.tbl returns 0, and the
problem was concurrent merge:
<details>
2022.01.16 09:07:34.289470 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Debug> executeQuery: (from [::1]:42082) (comment: 01615_random_one_shard_insertion.sql) select count() != 0 from shard_0.tbl;
2022.01.16 09:07:34.289564 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} <Debug> MergeTask::MergeProjectionsStage: Merge sorted 54 rows, containing 1 columns (1 merged, 0 gathered) in 0.00171193 sec., 31543.345814373253 rows/sec., 246.43 KiB>
2022.01.16 09:07:34.289810 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} <Trace> shard_0.tbl (c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab): Renaming temporary part tmp_merge_all_1_54_9 to all_1_54_9.
2022.01.16 09:07:34.289858 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> ContextAccess (default): Access granted: SELECT(number) ON shard_0.tbl
2022.01.16 09:07:34.289897 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} <Trace> shard_0.tbl (c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab) (MergerMutator): Merged 6 parts: from all_1_49_8 to all_54_54_0
2022.01.16 09:07:34.289920 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> InterpreterSelectQuery: WithMergeableState -> Complete
2022.01.16 09:07:34.289987 [ 375 ] {} <Debug> MemoryTracker: Peak memory usage Mutate/Merge: 3.12 MiB.
2022.01.16 09:07:34.290305 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> MergingAggregatedTransform: Reading blocks of partially aggregated data.
2022.01.16 09:07:34.290332 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Debug> MergingAggregatedTransform: Read 1 blocks of partially aggregated data, total 1 rows.
2022.01.16 09:07:34.290343 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> Aggregator: Merging partially aggregated single-level data.
2022.01.16 09:07:34.290358 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> Aggregator: Merged partially aggregated single-level data.
2022.01.16 09:07:34.290366 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Trace> Aggregator: Converting aggregated data to blocks
2022.01.16 09:07:34.290391 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Debug> Aggregator: Converted aggregated data to blocks. 1 rows, 8.00 B in 1.0939e-05 sec. (91416.034 rows/sec., 714.19 KiB/sec.)
2022.01.16 09:07:34.290709 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Information> executeQuery: Read 1 rows, 4.01 KiB in 0.001187722 sec., 841 rows/sec., 3.30 MiB/sec.
2022.01.16 09:07:34.290774 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} <Debug> MemoryTracker: Peak memory usage (for query): 0.00 B.
</details>
[1]: https://s3.amazonaws.com/clickhouse-test-reports/33675/7848ea7d609e4c720e8e4494eb6207c0751f5aea/stateless_tests__ubsan__actions_.html
This also fixes a race between DROP TABLE check and INSERT/merges.
v0: use Active parts instead.
v2: fix total counters accounting instead.
|
||
|---|---|---|
| .. | ||
| examples | ||
| tests | ||
| ActiveDataPartSet.cpp | ||
| ActiveDataPartSet.h | ||
| AllMergeSelector.cpp | ||
| AllMergeSelector.h | ||
| BackgroundJobsAssignee.cpp | ||
| BackgroundJobsAssignee.h | ||
| BackgroundProcessList.h | ||
| BoolMask.cpp | ||
| BoolMask.h | ||
| checkDataPart.cpp | ||
| checkDataPart.h | ||
| CMakeLists.txt | ||
| ColumnSizeEstimator.h | ||
| DataPartsExchange.cpp | ||
| DataPartsExchange.h | ||
| DropPartsRanges.cpp | ||
| DropPartsRanges.h | ||
| EphemeralLockInZooKeeper.cpp | ||
| EphemeralLockInZooKeeper.h | ||
| FutureMergedMutatedPart.cpp | ||
| FutureMergedMutatedPart.h | ||
| IExecutableTask.h | ||
| IMergedBlockOutputStream.cpp | ||
| IMergedBlockOutputStream.h | ||
| IMergeTreeDataPart.cpp | ||
| IMergeTreeDataPart.h | ||
| IMergeTreeDataPartWriter.cpp | ||
| IMergeTreeDataPartWriter.h | ||
| IMergeTreeReader.cpp | ||
| IMergeTreeReader.h | ||
| IntersectionsIndexes.h | ||
| KeyCondition.cpp | ||
| KeyCondition.h | ||
| LeaderElection.h | ||
| LevelMergeSelector.cpp | ||
| LevelMergeSelector.h | ||
| localBackup.cpp | ||
| localBackup.h | ||
| MarkRange.cpp | ||
| MarkRange.h | ||
| MergeAlgorithm.cpp | ||
| MergeAlgorithm.h | ||
| MergedBlockOutputStream.cpp | ||
| MergedBlockOutputStream.h | ||
| MergedColumnOnlyOutputStream.cpp | ||
| MergedColumnOnlyOutputStream.h | ||
| MergeFromLogEntryTask.cpp | ||
| MergeFromLogEntryTask.h | ||
| MergeList.cpp | ||
| MergeList.h | ||
| MergeMutateSelectedEntry.h | ||
| MergePlainMergeTreeTask.cpp | ||
| MergePlainMergeTreeTask.h | ||
| MergeProgress.h | ||
| MergeSelector.h | ||
| MergeTask.cpp | ||
| MergeTask.h | ||
| MergeTreeBackgroundExecutor.cpp | ||
| MergeTreeBackgroundExecutor.h | ||
| MergeTreeBaseSelectProcessor.cpp | ||
| MergeTreeBaseSelectProcessor.h | ||
| MergeTreeBlockReadUtils.cpp | ||
| MergeTreeBlockReadUtils.h | ||
| MergeTreeData.cpp | ||
| MergeTreeData.h | ||
| MergeTreeDataFormatVersion.h | ||
| MergeTreeDataMergerMutator.cpp | ||
| MergeTreeDataMergerMutator.h | ||
| MergeTreeDataPartChecksum.cpp | ||
| MergeTreeDataPartChecksum.h | ||
| MergeTreeDataPartCompact.cpp | ||
| MergeTreeDataPartCompact.h | ||
| MergeTreeDataPartInMemory.cpp | ||
| MergeTreeDataPartInMemory.h | ||
| MergeTreeDataPartTTLInfo.cpp | ||
| MergeTreeDataPartTTLInfo.h | ||
| MergeTreeDataPartType.cpp | ||
| MergeTreeDataPartType.h | ||
| MergeTreeDataPartUUID.cpp | ||
| MergeTreeDataPartUUID.h | ||
| MergeTreeDataPartWide.cpp | ||
| MergeTreeDataPartWide.h | ||
| MergeTreeDataPartWriterCompact.cpp | ||
| MergeTreeDataPartWriterCompact.h | ||
| MergeTreeDataPartWriterInMemory.cpp | ||
| MergeTreeDataPartWriterInMemory.h | ||
| MergeTreeDataPartWriterOnDisk.cpp | ||
| MergeTreeDataPartWriterOnDisk.h | ||
| MergeTreeDataPartWriterWide.cpp | ||
| MergeTreeDataPartWriterWide.h | ||
| MergeTreeDataSelectExecutor.cpp | ||
| MergeTreeDataSelectExecutor.h | ||
| MergeTreeDataWriter.cpp | ||
| MergeTreeDataWriter.h | ||
| MergeTreeDeduplicationLog.cpp | ||
| MergeTreeDeduplicationLog.h | ||
| MergeTreeIndexAggregatorBloomFilter.cpp | ||
| MergeTreeIndexAggregatorBloomFilter.h | ||
| MergeTreeIndexBloomFilter.cpp | ||
| MergeTreeIndexBloomFilter.h | ||
| MergeTreeIndexConditionBloomFilter.cpp | ||
| MergeTreeIndexConditionBloomFilter.h | ||
| MergeTreeIndexFullText.cpp | ||
| MergeTreeIndexFullText.h | ||
| MergeTreeIndexGranularity.cpp | ||
| MergeTreeIndexGranularity.h | ||
| MergeTreeIndexGranularityInfo.cpp | ||
| MergeTreeIndexGranularityInfo.h | ||
| MergeTreeIndexGranuleBloomFilter.cpp | ||
| MergeTreeIndexGranuleBloomFilter.h | ||
| MergeTreeIndexHypothesis.cpp | ||
| MergeTreeIndexHypothesis.h | ||
| MergeTreeIndexHypothesisMergedCondition.cpp | ||
| MergeTreeIndexHypothesisMergedCondition.h | ||
| MergeTreeIndexMinMax.cpp | ||
| MergeTreeIndexMinMax.h | ||
| MergeTreeIndexReader.cpp | ||
| MergeTreeIndexReader.h | ||
| MergeTreeIndexSet.cpp | ||
| MergeTreeIndexSet.h | ||
| MergeTreeIndices.cpp | ||
| MergeTreeIndices.h | ||
| MergeTreeInOrderSelectProcessor.cpp | ||
| MergeTreeInOrderSelectProcessor.h | ||
| MergeTreeIOSettings.h | ||
| MergeTreeMarksLoader.cpp | ||
| MergeTreeMarksLoader.h | ||
| MergeTreeMutationEntry.cpp | ||
| MergeTreeMutationEntry.h | ||
| MergeTreeMutationStatus.cpp | ||
| MergeTreeMutationStatus.h | ||
| MergeTreePartInfo.cpp | ||
| MergeTreePartInfo.h | ||
| MergeTreePartition.cpp | ||
| MergeTreePartition.h | ||
| MergeTreePartsMover.cpp | ||
| MergeTreePartsMover.h | ||
| MergeTreeRangeReader.cpp | ||
| MergeTreeRangeReader.h | ||
| MergeTreeReaderCompact.cpp | ||
| MergeTreeReaderCompact.h | ||
| MergeTreeReaderInMemory.cpp | ||
| MergeTreeReaderInMemory.h | ||
| MergeTreeReaderStream.cpp | ||
| MergeTreeReaderStream.h | ||
| MergeTreeReaderWide.cpp | ||
| MergeTreeReaderWide.h | ||
| MergeTreeReadPool.cpp | ||
| MergeTreeReadPool.h | ||
| MergeTreeReverseSelectProcessor.cpp | ||
| MergeTreeReverseSelectProcessor.h | ||
| MergeTreeSelectProcessor.cpp | ||
| MergeTreeSelectProcessor.h | ||
| MergeTreeSequentialSource.cpp | ||
| MergeTreeSequentialSource.h | ||
| MergeTreeSettings.cpp | ||
| MergeTreeSettings.h | ||
| MergeTreeSink.cpp | ||
| MergeTreeSink.h | ||
| MergeTreeThreadSelectProcessor.cpp | ||
| MergeTreeThreadSelectProcessor.h | ||
| MergeTreeWhereOptimizer.cpp | ||
| MergeTreeWhereOptimizer.h | ||
| MergeTreeWriteAheadLog.cpp | ||
| MergeTreeWriteAheadLog.h | ||
| MergeType.cpp | ||
| MergeType.h | ||
| MutateFromLogEntryTask.cpp | ||
| MutateFromLogEntryTask.h | ||
| MutatePlainMergeTreeTask.cpp | ||
| MutatePlainMergeTreeTask.h | ||
| MutateTask.cpp | ||
| MutateTask.h | ||
| ParallelReplicasReadingCoordinator.cpp | ||
| ParallelReplicasReadingCoordinator.h | ||
| PartitionPruner.cpp | ||
| PartitionPruner.h | ||
| PartMovesBetweenShardsOrchestrator.cpp | ||
| PartMovesBetweenShardsOrchestrator.h | ||
| PinnedPartUUIDs.cpp | ||
| PinnedPartUUIDs.h | ||
| RangesInDataPart.h | ||
| registerStorageMergeTree.cpp | ||
| ReplicatedFetchList.cpp | ||
| ReplicatedFetchList.h | ||
| ReplicatedMergeMutateTaskBase.cpp | ||
| ReplicatedMergeMutateTaskBase.h | ||
| ReplicatedMergeTreeAddress.cpp | ||
| ReplicatedMergeTreeAddress.h | ||
| ReplicatedMergeTreeAltersSequence.cpp | ||
| ReplicatedMergeTreeAltersSequence.h | ||
| ReplicatedMergeTreeCleanupThread.cpp | ||
| ReplicatedMergeTreeCleanupThread.h | ||
| ReplicatedMergeTreeLogEntry.cpp | ||
| ReplicatedMergeTreeLogEntry.h | ||
| ReplicatedMergeTreeMergeStrategyPicker.cpp | ||
| ReplicatedMergeTreeMergeStrategyPicker.h | ||
| ReplicatedMergeTreeMutationEntry.cpp | ||
| ReplicatedMergeTreeMutationEntry.h | ||
| ReplicatedMergeTreePartCheckThread.cpp | ||
| ReplicatedMergeTreePartCheckThread.h | ||
| ReplicatedMergeTreePartHeader.cpp | ||
| ReplicatedMergeTreePartHeader.h | ||
| ReplicatedMergeTreeQueue.cpp | ||
| ReplicatedMergeTreeQueue.h | ||
| ReplicatedMergeTreeQuorumAddedParts.h | ||
| ReplicatedMergeTreeQuorumEntry.h | ||
| ReplicatedMergeTreeRestartingThread.cpp | ||
| ReplicatedMergeTreeRestartingThread.h | ||
| ReplicatedMergeTreeSink.cpp | ||
| ReplicatedMergeTreeSink.h | ||
| ReplicatedMergeTreeTableMetadata.cpp | ||
| ReplicatedMergeTreeTableMetadata.h | ||
| RequestResponse.cpp | ||
| RequestResponse.h | ||
| RPNBuilder.h | ||
| SimpleMergeSelector.cpp | ||
| SimpleMergeSelector.h | ||
| StorageFromMergeTreeDataPart.h | ||
| TTLMergeSelector.cpp | ||
| TTLMergeSelector.h | ||
| ZeroCopyLock.cpp | ||
| ZeroCopyLock.h | ||