From 4fe0e7be85002ac716fddaeaaab06aa08d3cb0e5 Mon Sep 17 00:00:00 2001 From: ruoxi Date: Tue, 23 Jul 2019 13:00:18 +0800 Subject: [PATCH] Add rename/truncate on read/write test and alter on read test (#128) * Add sync schema on read * Simplify schema syncer interface and adjust mock stuff * Rename default schema version setting * Compensate last commit * Remove curl library * Remove curl from builder image * Remove useless codes, init schema syncer based on pd config * Minor fix to schema debug * Fix alter tmt and pass tests * Fix build fail * Add lock for mock schema syncer * Fix schema sync service init context * Adjust schema tests * Not sync if no schema change detected * Adjust txn mock tests * Fix default value bug * Rename some tests * Remove sync schema test * Remove a lot useless code * Refine schema sync on read, and add drop on read test * Support rename mock tidb table * Add rename tests * Add mock truncate table and tests * Add rename/truncate on read/write test and alter on read test --- contrib/client-c/include/pd/MockPDClient.h | 42 ++--- dbms/src/Debug/MockTiDB.cpp | 6 +- dbms/src/Debug/MockTiDB.h | 4 +- dbms/src/Debug/dbgFuncMockTiDBTable.cpp | 8 +- .../txn_schema/alter_on_read.test | 174 ++++++++++++++++++ .../txn_schema/alter_on_write.test | 4 +- 6 files changed, 202 insertions(+), 36 deletions(-) create mode 100644 tests/mutable-test/txn_schema/alter_on_read.test diff --git a/contrib/client-c/include/pd/MockPDClient.h b/contrib/client-c/include/pd/MockPDClient.h index 89b41898c9b..6d0cd1e3e40 100644 --- a/contrib/client-c/include/pd/MockPDClient.h +++ b/contrib/client-c/include/pd/MockPDClient.h @@ -1,46 +1,34 @@ #pragma once -#include #include +#include -namespace pingcap { -namespace pd { +namespace pingcap +{ +namespace pd +{ using Clock = std::chrono::system_clock; -using Seconds = std::chrono::seconds; -class MockPDClient : public IClient { +class MockPDClient : public IClient +{ public: MockPDClient() = default; ~MockPDClient() override {} - uint64_t getGCSafePoint() override - { - return (Clock::now() - Seconds(2)).time_since_epoch().count(); - } + uint64_t getGCSafePoint() override { return 1000000000000000; } - uint64_t getTS() override - { - return Clock::now().time_since_epoch().count(); - } + uint64_t getTS() override { return Clock::now().time_since_epoch().count(); } - std::tuple> getRegion(std::string) override { - throw "not implemented"; - } + std::tuple> getRegion(std::string) override { throw "not implemented"; } - std::tuple> getRegionByID(uint64_t) override { - throw "not implemented"; - } + std::tuple> getRegionByID(uint64_t) override { throw "not implemented"; } - metapb::Store getStore(uint64_t) override { - throw "not implemented"; - } + metapb::Store getStore(uint64_t) override { throw "not implemented"; } - bool isMock() override { - return true; - } + bool isMock() override { return true; } }; -} -} +} // namespace pd +} // namespace pingcap diff --git a/dbms/src/Debug/MockTiDB.cpp b/dbms/src/Debug/MockTiDB.cpp index e6692ab59c1..0dda387b43f 100644 --- a/dbms/src/Debug/MockTiDB.cpp +++ b/dbms/src/Debug/MockTiDB.cpp @@ -104,7 +104,7 @@ ColumnInfo getColumnInfoFromColumn(const NameAndTypePair & column, ColumnID id) return column_info; } -TableID MockTiDB::newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns) +TableID MockTiDB::newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso) { std::lock_guard lock(tables_mutex); @@ -133,6 +133,7 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na table_info.pk_is_handle = false; table_info.comment = "Mocked."; + table_info.update_timestamp = tso; auto table = std::make_shared(database_name, table_name, std::move(table_info)); tables_by_id.emplace(table->table_info.id, table); @@ -141,7 +142,7 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na return table->table_info.id; } -TableID MockTiDB::newPartition(const String & database_name, const String & table_name, const String & partition_name) +TableID MockTiDB::newPartition(const String & database_name, const String & table_name, const String & partition_name, Timestamp tso) { std::lock_guard lock(tables_mutex); @@ -162,6 +163,7 @@ TableID MockTiDB::newPartition(const String & database_name, const String & tabl partition_def.id = partition_id; partition_def.name = partition_name; table_info.partition.definitions.emplace_back(partition_def); + table_info.update_timestamp = tso; // Map the same table object with partition ID as key, so mock schema syncer behaves the same as TiDB, // i.e. gives the table info by partition ID. diff --git a/dbms/src/Debug/MockTiDB.h b/dbms/src/Debug/MockTiDB.h index f60c0fe8b59..6b5186ba603 100644 --- a/dbms/src/Debug/MockTiDB.h +++ b/dbms/src/Debug/MockTiDB.h @@ -62,9 +62,9 @@ class MockTiDB : public ext::singleton using TablePtr = std::shared_ptr
; public: - TableID newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns); + TableID newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso); - TableID newPartition(const String & database_name, const String & table_name, const String & partition_name); + TableID newPartition(const String & database_name, const String & table_name, const String & partition_name, Timestamp tso); void dropTable(const String & database_name, const String & table_name); diff --git a/dbms/src/Debug/dbgFuncMockTiDBTable.cpp b/dbms/src/Debug/dbgFuncMockTiDBTable.cpp index 2db846688b9..e2b961ba4fc 100644 --- a/dbms/src/Debug/dbgFuncMockTiDBTable.cpp +++ b/dbms/src/Debug/dbgFuncMockTiDBTable.cpp @@ -41,15 +41,16 @@ void MockTiDBTable::dbgFuncMockTiDBTable(Context & context, const ASTs & args, D throw Exception("Invalid TiDB table schema", ErrorCodes::LOGICAL_ERROR); ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(typeid_cast(*columns_ast), context); + auto tso = context.getTMTContext().getPDClient()->getTS(); - TableID table_id = MockTiDB::instance().newTable(database_name, table_name, columns); + TableID table_id = MockTiDB::instance().newTable(database_name, table_name, columns, tso); std::stringstream ss; ss << "mock table #" << table_id; output(ss.str()); } -void MockTiDBTable::dbgFuncMockTiDBPartition(Context &, const ASTs & args, DBGInvoker::Printer output) +void MockTiDBTable::dbgFuncMockTiDBPartition(Context & context, const ASTs & args, DBGInvoker::Printer output) { if (args.size() != 3) throw Exception("Args not matched, should be: database-name, table-name, partition-name", ErrorCodes::BAD_ARGUMENTS); @@ -57,8 +58,9 @@ void MockTiDBTable::dbgFuncMockTiDBPartition(Context &, const ASTs & args, DBGIn const String & database_name = typeid_cast(*args[0]).name; const String & table_name = typeid_cast(*args[1]).name; const String & partition_name = typeid_cast(*args[2]).name; + auto tso = context.getTMTContext().getPDClient()->getTS(); - TableID partition_id = MockTiDB::instance().newPartition(database_name, table_name, partition_name); + TableID partition_id = MockTiDB::instance().newPartition(database_name, table_name, partition_name, tso); std::stringstream ss; ss << "mock partition #" << partition_id; diff --git a/tests/mutable-test/txn_schema/alter_on_read.test b/tests/mutable-test/txn_schema/alter_on_read.test new file mode 100644 index 00000000000..1ffdb492aa8 --- /dev/null +++ b/tests/mutable-test/txn_schema/alter_on_read.test @@ -0,0 +1,174 @@ +# Preparation. +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __set_flush_threshold(1000000, 1000000) +=> DBGInvoke __mock_schema_syncer('true') + +=> DBGInvoke __enable_schema_sync_service('false') + +# Sync add column by reading. +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __put_region(4, 0, 100, default, test) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') +=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1) +# Data in region cache with tso greater than read tso will be ignored. +=> select * from default.test " --read_tso "1500000000000000 +# Data in region cache with tso less than read tso will be force decoded (extra column will be discarded), even under an out-of-date schema. +=> select col_1 from default.test +┌─col_1─┐ +│ test1 │ +└───────┘ +# Verify this schema is truely out-of-date. +=> select col_2 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "100 +┌─col_2─┐ +│ 1 │ +└───────┘ + +# Sync drop column by reading. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1) +=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2) +# Data in region cache with tso less than read tso will be force decoded (missing column will be filled with default value or null), even under an out-of-date schema. +=> select col_1 from default.test +┌─col_1─┐ +│ test1 │ +│ │ +└───────┘ +# Read with specified bigger schema version will trigger schema sync. +=> select col_1 from default.test " --schema_version "100 +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_1. + +# Sync type change by checking sign overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int16)') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128) +=> select col_2 from default.test +┌─col_2─┐ +│ 1 │ +│ 2 │ +│ -128 │ +└───────┘ +=> desc default.test +┌─name────────┬─type───────────┬─default_type─┬─default_expression─┐ +│ _tidb_rowid │ Int64 │ │ │ +│ col_2 │ Nullable(Int8) │ │ │ +└─────────────┴────────────────┴──────────────┴────────────────────┘ +=> DBGInvoke __raft_insert_row(default, test, 4, 53, 128) +# 128 will overflow when decoding using out-of-date schema (Int8). +=> select col_2 from default.test +Received exception from server (version {#WORD}): +Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 128 of type Nullable(Int8). +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "100 +┌─col_2─┐ +│ 1 │ +│ 2 │ +│ -128 │ +│ 128 │ +└───────┘ + +# Sync type change by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 54, 65536) +# 65536 will overflow when decoding using out-of-date schema (Int16). +=> select col_2 from default.test +Received exception from server (version {#WORD}): +Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 65536 of type Nullable(Int16). +# Read with specified bigger schema version will trigger schema sync. +=> select col_2 from default.test " --schema_version "100 +┌─col_2─┐ +│ 1 │ +│ 2 │ +│ -128 │ +│ 128 │ +│ 65536 │ +└───────┘ + +# Sync add column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 UInt8') +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 UInt64') +=> DBGInvoke __raft_insert_row(default, test, 4, 55, 0, 256) +=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 Nullable(UInt8)') +# 256 will overflow when decoding using out-of-date schema (UInt8). +=> select col_3 from default.test +Received exception from server (version {#WORD}): +Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type UInt8. +# Read with specified bigger schema version will trigger schema sync. +=> select col_3, col_4 from default.test " --schema_version "100 +┌─col_3─┬─col_4─┐ +│ 0 │ \N │ +│ 0 │ \N │ +│ 0 │ \N │ +│ 0 │ \N │ +│ 0 │ \N │ +│ 256 │ \N │ +└───────┴───────┘ + +# Edge values not overflowing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_3) +=> select col_2, col_3, col_4 from default.test +┌─col_2─┬─col_3─┬─col_4─┐ +│ 1 │ 0 │ \N │ +│ 2 │ 0 │ \N │ +└───────┴───────┴───────┘ +┌─col_2─┬─col_3─┬─col_4─┐ +│ -128 │ 0 │ \N │ +│ 128 │ 0 │ \N │ +│ 65536 │ 0 │ \N │ +└───────┴───────┴───────┘ +┌────────────────col_2─┬────────────────col_3─┬─col_4─┐ +│ 0 │ 256 │ \N │ +│ -9223372036854775807 │ 18446744073709551615 │ 1 │ +└──────────────────────┴──────────────────────┴───────┘ +┌───────────────col_2─┬────────────────col_3─┬─col_4─┐ +│ 9223372036854775807 │ 18446744073709551615 │ 1 │ +└─────────────────────┴──────────────────────┴───────┘ +=> select col_3, col_4 from default.test " --schema_version "100 +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_3. + +# Sync drop column and type change together by checking value overflow in CH when flushing. +=> DBGInvoke __try_flush_region(4) +=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)') +=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256) +=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2) +# -256 will overflow when decoding using out-of-date schema (UInt8). +=> select col_4 from default.test +Received exception from server (version {#WORD}): +Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type Nullable(UInt8). +# Read with specified bigger schema version will trigger schema sync. +=> select col_4 from default.test " --schema_version "100 +┌─col_4─┐ +│ \N │ +│ \N │ +│ \N │ +│ \N │ +│ \N │ +│ \N │ +│ 1 │ +│ 1 │ +│ 256 │ +└───────┘ +=> select col_2 from default.test +Received exception from server (version {#WORD}): +Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. + +# Clean up. +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test +=> DBGInvoke __enable_schema_sync_service('true') diff --git a/tests/mutable-test/txn_schema/alter_on_write.test b/tests/mutable-test/txn_schema/alter_on_write.test index 3a94cce1c22..dc9b59ed968 100644 --- a/tests/mutable-test/txn_schema/alter_on_write.test +++ b/tests/mutable-test/txn_schema/alter_on_write.test @@ -94,7 +94,7 @@ Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier └───────┴───────┘ # Not sync drop column for edge values not overflowing. -=> DBGInvoke __raft_insert_row(default, test, 4, 55, -9223372036854775807, 18446744073709551615, 1) +=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1) => DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1) => DBGInvoke __drop_column_from_tidb_table(default, test, col_3) => DBGInvoke __try_flush_region(4) @@ -120,7 +120,7 @@ Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier # Sync drop column and type change together by checking value overflow in CH when flushing. => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)') -=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, -256) +=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256) => DBGInvoke __drop_column_from_tidb_table(default, test, col_2) => desc default.test ┌─name────────┬─type────────────┬─default_type─┬─default_expression─┐