-
Notifications
You must be signed in to change notification settings - Fork 409
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add rename/truncate on read/write test and alter on read test (#128)
* Add sync schema on read * Simplify schema syncer interface and adjust mock stuff * Rename default schema version setting * Compensate last commit * Remove curl library * Remove curl from builder image * Remove useless codes, init schema syncer based on pd config * Minor fix to schema debug * Fix alter tmt and pass tests * Fix build fail * Add lock for mock schema syncer * Fix schema sync service init context * Adjust schema tests * Not sync if no schema change detected * Adjust txn mock tests * Fix default value bug * Rename some tests * Remove sync schema test * Remove a lot useless code * Refine schema sync on read, and add drop on read test * Support rename mock tidb table * Add rename tests * Add mock truncate table and tests * Add rename/truncate on read/write test and alter on read test
- Loading branch information
1 parent
bc79718
commit 4fe0e7b
Showing
6 changed files
with
202 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,46 +1,34 @@ | ||
| #pragma once | ||
|
|
||
| #include <limits> | ||
| #include <pd/IClient.h> | ||
| #include <limits> | ||
|
|
||
| namespace pingcap { | ||
| namespace pd { | ||
| namespace pingcap | ||
| { | ||
| namespace pd | ||
| { | ||
|
|
||
| using Clock = std::chrono::system_clock; | ||
| using Seconds = std::chrono::seconds; | ||
|
|
||
| class MockPDClient : public IClient { | ||
| class MockPDClient : public IClient | ||
| { | ||
| public: | ||
| MockPDClient() = default; | ||
|
|
||
| ~MockPDClient() override {} | ||
|
|
||
| uint64_t getGCSafePoint() override | ||
| { | ||
| return (Clock::now() - Seconds(2)).time_since_epoch().count(); | ||
| } | ||
| uint64_t getGCSafePoint() override { return 1000000000000000; } | ||
|
|
||
| uint64_t getTS() override | ||
| { | ||
| return Clock::now().time_since_epoch().count(); | ||
| } | ||
| uint64_t getTS() override { return Clock::now().time_since_epoch().count(); } | ||
|
|
||
| std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegion(std::string) override { | ||
| throw "not implemented"; | ||
| } | ||
| std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegion(std::string) override { throw "not implemented"; } | ||
|
|
||
| std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegionByID(uint64_t) override { | ||
| throw "not implemented"; | ||
| } | ||
| std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegionByID(uint64_t) override { throw "not implemented"; } | ||
|
|
||
| metapb::Store getStore(uint64_t) override { | ||
| throw "not implemented"; | ||
| } | ||
| metapb::Store getStore(uint64_t) override { throw "not implemented"; } | ||
|
|
||
| bool isMock() override { | ||
| return true; | ||
| } | ||
| bool isMock() override { return true; } | ||
| }; | ||
|
|
||
| } | ||
| } | ||
| } // namespace pd | ||
| } // namespace pingcap |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| # Preparation. | ||
| => DBGInvoke __enable_schema_sync_service('false') | ||
|
|
||
| => DBGInvoke __drop_tidb_table(default, test) | ||
| => drop table if exists default.test | ||
|
|
||
| => DBGInvoke __set_flush_threshold(1000000, 1000000) | ||
| => DBGInvoke __mock_schema_syncer('true') | ||
|
|
||
| => DBGInvoke __enable_schema_sync_service('false') | ||
|
|
||
| # Sync add column by reading. | ||
| => DBGInvoke __mock_tidb_table(default, test, 'col_1 String') | ||
| => DBGInvoke __refresh_schemas() | ||
| => DBGInvoke __put_region(4, 0, 100, default, test) | ||
| => DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)') | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1) | ||
| # Data in region cache with tso greater than read tso will be ignored. | ||
| => select * from default.test " --read_tso "1500000000000000 | ||
| # Data in region cache with tso less than read tso will be force decoded (extra column will be discarded), even under an out-of-date schema. | ||
| => select col_1 from default.test | ||
| ┌─col_1─┐ | ||
| │ test1 │ | ||
| └───────┘ | ||
| # Verify this schema is truely out-of-date. | ||
| => select col_2 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_2 from default.test " --schema_version "100 | ||
| ┌─col_2─┐ | ||
| │ 1 │ | ||
| └───────┘ | ||
|
|
||
| # Sync drop column by reading. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __drop_column_from_tidb_table(default, test, col_1) | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 51, 2) | ||
| # Data in region cache with tso less than read tso will be force decoded (missing column will be filled with default value or null), even under an out-of-date schema. | ||
| => select col_1 from default.test | ||
| ┌─col_1─┐ | ||
| │ test1 │ | ||
| │ │ | ||
| └───────┘ | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_1 from default.test " --schema_version "100 | ||
| Received exception from server (version {#WORD}): | ||
| Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_1. | ||
|
|
||
| # Sync type change by checking sign overflow in CH when flushing. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int16)') | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 52, -128) | ||
| => select col_2 from default.test | ||
| ┌─col_2─┐ | ||
| │ 1 │ | ||
| │ 2 │ | ||
| │ -128 │ | ||
| └───────┘ | ||
| => desc default.test | ||
| ┌─name────────┬─type───────────┬─default_type─┬─default_expression─┐ | ||
| │ _tidb_rowid │ Int64 │ │ │ | ||
| │ col_2 │ Nullable(Int8) │ │ │ | ||
| └─────────────┴────────────────┴──────────────┴────────────────────┘ | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 53, 128) | ||
| # 128 will overflow when decoding using out-of-date schema (Int8). | ||
| => select col_2 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 128 of type Nullable(Int8). | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_2 from default.test " --schema_version "100 | ||
| ┌─col_2─┐ | ||
| │ 1 │ | ||
| │ 2 │ | ||
| │ -128 │ | ||
| │ 128 │ | ||
| └───────┘ | ||
|
|
||
| # Sync type change by checking value overflow in CH when flushing. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)') | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 54, 65536) | ||
| # 65536 will overflow when decoding using out-of-date schema (Int16). | ||
| => select col_2 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 65536 of type Nullable(Int16). | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_2 from default.test " --schema_version "100 | ||
| ┌─col_2─┐ | ||
| │ 1 │ | ||
| │ 2 │ | ||
| │ -128 │ | ||
| │ 128 │ | ||
| │ 65536 │ | ||
| └───────┘ | ||
|
|
||
| # Sync add column and type change together by checking value overflow in CH when flushing. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 UInt8') | ||
| => DBGInvoke __refresh_schemas() | ||
| => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 UInt64') | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 55, 0, 256) | ||
| => DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 Nullable(UInt8)') | ||
| # 256 will overflow when decoding using out-of-date schema (UInt8). | ||
| => select col_3 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type UInt8. | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_3, col_4 from default.test " --schema_version "100 | ||
| ┌─col_3─┬─col_4─┐ | ||
| │ 0 │ \N │ | ||
| │ 0 │ \N │ | ||
| │ 0 │ \N │ | ||
| │ 0 │ \N │ | ||
| │ 0 │ \N │ | ||
| │ 256 │ \N │ | ||
| └───────┴───────┘ | ||
|
|
||
| # Edge values not overflowing. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1) | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1) | ||
| => DBGInvoke __drop_column_from_tidb_table(default, test, col_3) | ||
| => select col_2, col_3, col_4 from default.test | ||
| ┌─col_2─┬─col_3─┬─col_4─┐ | ||
| │ 1 │ 0 │ \N │ | ||
| │ 2 │ 0 │ \N │ | ||
| └───────┴───────┴───────┘ | ||
| ┌─col_2─┬─col_3─┬─col_4─┐ | ||
| │ -128 │ 0 │ \N │ | ||
| │ 128 │ 0 │ \N │ | ||
| │ 65536 │ 0 │ \N │ | ||
| └───────┴───────┴───────┘ | ||
| ┌────────────────col_2─┬────────────────col_3─┬─col_4─┐ | ||
| │ 0 │ 256 │ \N │ | ||
| │ -9223372036854775807 │ 18446744073709551615 │ 1 │ | ||
| └──────────────────────┴──────────────────────┴───────┘ | ||
| ┌───────────────col_2─┬────────────────col_3─┬─col_4─┐ | ||
| │ 9223372036854775807 │ 18446744073709551615 │ 1 │ | ||
| └─────────────────────┴──────────────────────┴───────┘ | ||
| => select col_3, col_4 from default.test " --schema_version "100 | ||
| Received exception from server (version {#WORD}): | ||
| Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_3. | ||
|
|
||
| # Sync drop column and type change together by checking value overflow in CH when flushing. | ||
| => DBGInvoke __try_flush_region(4) | ||
| => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)') | ||
| => DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256) | ||
| => DBGInvoke __drop_column_from_tidb_table(default, test, col_2) | ||
| # -256 will overflow when decoding using out-of-date schema (UInt8). | ||
| => select col_4 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type Nullable(UInt8). | ||
| # Read with specified bigger schema version will trigger schema sync. | ||
| => select col_4 from default.test " --schema_version "100 | ||
| ┌─col_4─┐ | ||
| │ \N │ | ||
| │ \N │ | ||
| │ \N │ | ||
| │ \N │ | ||
| │ \N │ | ||
| │ \N │ | ||
| │ 1 │ | ||
| │ 1 │ | ||
| │ 256 │ | ||
| └───────┘ | ||
| => select col_2 from default.test | ||
| Received exception from server (version {#WORD}): | ||
| Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2. | ||
|
|
||
| # Clean up. | ||
| => DBGInvoke __drop_tidb_table(default, test) | ||
| => drop table if exists default.test | ||
| => DBGInvoke __enable_schema_sync_service('true') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters