thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

done

Browse Source
This commit is contained in:
Nikita Mikhaylov 2021-06-10 13:57:19 +00:00
parent 4185c5954f
commit 395e0d6756
5 changed files with 31 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/Processors/Formats/IInputFormat.h
View File

@ -12,7 +12,7 @@ struct ColumnMapping
{
/// Non-atomic because there is strict `happens-before` between read and write access
/// See InputFormatParallelParsing
bool is_set;
bool is_set{false};
/// Maps indexes of columns in the input file to indexes of table columns
using OptionalIndexes = std::vector<std::optional<size_t>>;
OptionalIndexes column_indexes_for_input_fields;
@ -22,6 +22,11 @@ struct ColumnMapping
/// read the file header, and never changed afterwards.
/// For other columns, it is updated on each read() call.
std::vector<UInt8> read_columns;
/// Whether we have any columns that are not read from file at all,
/// and must be always initialized with defaults.
bool have_always_default_columns{false};
};
using ColumnMappingPtr = std::shared_ptr<ColumnMapping>;

6
src/Processors/Formats/Impl/CSVRowInputFormat.cpp
View File

@ -193,7 +193,7 @@ void CSVRowInputFormat::readPrefix()
{
if (!read_column)
{
have_always_default_columns = true;
column_mapping->have_always_default_columns = true;
break;
}
}
@ -221,7 +221,7 @@ bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext
/// Track whether we have to fill any columns in this row with default
/// values. If not, we return an empty column mask to the caller, so that
/// it doesn't have to check it.
bool have_default_columns = have_always_default_columns;
bool have_default_columns = column_mapping->have_always_default_columns;
ext.read_columns.assign(column_mapping->read_columns.size(), true);
const auto delimiter = format_settings.csv.delimiter;
@ -416,7 +416,7 @@ void CSVRowInputFormat::resetParser()
RowInputFormatWithDiagnosticInfo::resetParser();
column_mapping->column_indexes_for_input_fields.clear();
column_mapping->read_columns.clear();
have_always_default_columns = false;
column_mapping->have_always_default_columns = false;
}

6
src/Processors/Formats/Impl/CSVRowInputFormat.h
View File

@ -31,17 +31,13 @@ public:
void resetParser() override;
private:
/// There fields are computed in constructor.
bool with_names;
const FormatSettings format_settings;
DataTypes data_types;
using IndexesMap = std::unordered_map<String, size_t>;
IndexesMap column_indexes_by_names;
/// Whether we have any columns that are not read from file at all,
/// and must be always initialized with defaults.
bool have_always_default_columns = false;
void addInputColumn(const String & column_name);
void setupAllColumnsByTableSchema();

2
tests/queries/0_stateless/01903_csvwithnames_subset_of_columns.reference Normal file
View File

@ -0,0 +1,2 @@
1000000
2000000

19
tests/queries/0_stateless/01903_csvwithnames_subset_of_columns.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_01903"
$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01903 (col0 Date, col1 Nullable(UInt8)) ENGINE MergeTree() PARTITION BY toYYYYMM(col0) ORDER BY col0;"
(echo col0,col1; for _ in `seq 1 1000000`; do echo '2021-05-05',1; done) | $CLICKHOUSE_CLIENT -q "INSERT INTO test_01903 FORMAT CSVWithNames"
$CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01903"
(echo col0; for _ in `seq 1 1000000`; do echo '2021-05-05'; done) | $CLICKHOUSE_CLIENT -q "INSERT INTO test_01903 (col0) FORMAT CSVWithNames"
$CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01903"
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_01903"