Skip to content

Commit 1b40aba

Browse files
JaySon-HuangJaySon-Huang
authored
[FLASH-480] Fix bug: rename column (#227)
* fix bug: rename column * log info instead of error when column rename is detected
1 parent 96b4f0e commit 1b40aba

File tree

5 files changed

+244
-83
lines changed

5 files changed

+244
-83
lines changed

‎dbms/src/Storages/Transaction/SchemaBuilder-internal.h

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#pragma once
2+
3+
#include <map>
4+
#include <set>
5+
#include <utility>
6+
7+
#include <Core/Types.h>
8+
#include <Storages/Transaction/Types.h>
9+
10+
/// === Some Private struct / method for SchemaBuilder
11+
/// Notice that this file should only included by SchemaBuilder.cpp and unittest for this file.
12+
13+
namespace DB
14+
{
15+
16+
constexpr char tmpNamePrefix[] = "_tiflash_tmp_";
17+
18+
struct TmpTableNameGenerator
19+
{
20+
using TableName = std::pair<String, String>;
21+
TableName operator()(const TableName & name) { return std::make_pair(name.first, String(tmpNamePrefix) + name.second); }
22+
};
23+
24+
struct TmpColNameGenerator
25+
{
26+
String operator()(const String & name) { return String(tmpNamePrefix) + name; }
27+
};
28+
29+
30+
struct ColumnNameWithID
31+
{
32+
String name;
33+
ColumnID id;
34+
35+
explicit ColumnNameWithID(String name_ = "", ColumnID id_ = 0) : name(std::move(name_)), id(id_) {}
36+
37+
bool operator==(const ColumnNameWithID & rhs) const { return name == rhs.name; }
38+
39+
bool operator<(const ColumnNameWithID & rhs) const { return name < rhs.name; }
40+
};
41+
42+
struct TmpColNameWithIDGenerator
43+
{
44+
ColumnNameWithID operator()(const ColumnNameWithID & name_with_id)
45+
{
46+
return ColumnNameWithID{String(tmpNamePrefix) + name_with_id.name, name_with_id.id};
47+
}
48+
};
49+
50+
51+
// CyclicRenameResolver resolves cyclic table rename and column rename.
52+
// TmpNameGenerator rename current name to a temp name that will not conflict with other names.
53+
template <typename Name_, typename TmpNameGenerator>
54+
struct CyclicRenameResolver
55+
{
56+
using Name = Name_;
57+
using NamePair = std::pair<Name, Name>;
58+
using NamePairs = std::vector<NamePair>;
59+
using NameSet = std::set<Name>;
60+
using NameMap = std::map<Name, Name>;
61+
62+
// visited records which name has been processed.
63+
NameSet visited;
64+
TmpNameGenerator name_gen;
65+
66+
// We will not ensure correctness if we call it multiple times, so we make it a rvalue call.
67+
NamePairs resolve(NameMap && rename_map) &&
68+
{
69+
NamePairs result;
70+
for (auto it = rename_map.begin(); it != rename_map.end(); /* */)
71+
{
72+
if (!visited.count(it->first))
73+
{
74+
resolveImpl(rename_map, it, result);
75+
}
76+
// remove dependency of `it` since we have already done rename
77+
it = rename_map.erase(it);
78+
}
79+
return result;
80+
}
81+
82+
private:
83+
NamePair resolveImpl(NameMap & rename_map, typename NameMap::iterator & it, NamePairs & result)
84+
{
85+
Name origin_name = it->first;
86+
Name target_name = it->second;
87+
visited.insert(it->first);
88+
auto next_it = rename_map.find(target_name);
89+
if (next_it == rename_map.end())
90+
{
91+
// The target name does not exist, so we can rename it directly.
92+
result.push_back(NamePair(origin_name, target_name));
93+
return NamePair();
94+
}
95+
else if (visited.find(target_name) != visited.end())
96+
{
97+
// The target name is visited, so this is a cyclic rename.
98+
auto tmp_name = name_gen(target_name);
99+
result.push_back(NamePair(target_name, tmp_name));
100+
result.push_back(NamePair(origin_name, target_name));
101+
return NamePair(target_name, tmp_name);
102+
}
103+
else
104+
{
105+
// The target name is in rename map, so we continue to resolve it.
106+
auto pair = resolveImpl(rename_map, next_it, result);
107+
if (pair.first == origin_name)
108+
{
109+
origin_name = pair.second;
110+
}
111+
result.push_back(NamePair(origin_name, target_name));
112+
return pair;
113+
}
114+
}
115+
};
116+
117+
118+
} // namespace DB

‎dbms/src/Storages/Transaction/SchemaBuilder.cpp

+7-83
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <Parsers/parseQuery.h>
1414
#include <Storages/MutableSupport.h>
1515
#include <Storages/Transaction/SchemaBuilder.h>
16+
#include <Storages/Transaction/SchemaBuilder-internal.h>
1617
#include <Storages/Transaction/TMTContext.h>
1718
#include <Storages/Transaction/TypeMapping.h>
1819

@@ -26,84 +27,6 @@ namespace ErrorCodes
2627
extern const int DDL_ERROR;
2728
}
2829

29-
30-
constexpr char tmpNamePrefix[] = "_tiflash_tmp_";
31-
32-
struct TmpTableNameGenerator
33-
{
34-
using TableName = std::pair<String, String>;
35-
TableName operator()(const TableName & name) { return std::make_pair(name.first, String(tmpNamePrefix) + name.second); }
36-
};
37-
38-
struct TmpColNameGenerator
39-
{
40-
String operator()(const String & name) { return String(tmpNamePrefix) + name; }
41-
};
42-
43-
// CyclicRenameResolver resolves cyclic table rename and column rename.
44-
// TmpNameGenerator rename current name to a temp name that will not conflict with other names.
45-
template <typename Name_, typename TmpNameGenerator>
46-
struct CyclicRenameResolver
47-
{
48-
using Name = Name_;
49-
using NamePair = std::pair<Name, Name>;
50-
using NameMap = std::map<Name, Name>;
51-
using NameSet = std::set<Name>;
52-
53-
// visited records which name has been processed.
54-
NameSet visited;
55-
TmpNameGenerator name_gen;
56-
57-
// We will not ensure correctness if we call it multiple times, so we make it a rvalue call.
58-
std::vector<NamePair> resolve(const NameMap & rename_map) &&
59-
{
60-
std::vector<NamePair> result;
61-
for (auto it = rename_map.begin(); it != rename_map.end(); it++)
62-
{
63-
if (!visited.count(it->first))
64-
{
65-
resolveImpl(rename_map, it, result);
66-
}
67-
}
68-
return result;
69-
}
70-
71-
private:
72-
NamePair resolveImpl(const NameMap & rename_map, typename NameMap::const_iterator & it, std::vector<NamePair> & result)
73-
{
74-
Name target_name = it->second;
75-
Name origin_name = it->first;
76-
visited.insert(it->first);
77-
auto next_it = rename_map.find(target_name);
78-
if (next_it == rename_map.end())
79-
{
80-
// The target name does not exist, so we can rename it directly.
81-
result.push_back(NamePair(origin_name, target_name));
82-
return NamePair();
83-
}
84-
else if (visited.find(target_name) != visited.end())
85-
{
86-
// The target name is visited, so this is a cyclic rename.
87-
auto tmp_name = name_gen(target_name);
88-
result.push_back(NamePair(target_name, tmp_name));
89-
result.push_back(NamePair(origin_name, target_name));
90-
return NamePair(target_name, tmp_name);
91-
}
92-
else
93-
{
94-
// The target name is in rename map, so we continue to resolve it.
95-
auto pair = resolveImpl(rename_map, next_it, result);
96-
if (pair.first == origin_name)
97-
{
98-
origin_name = pair.second;
99-
}
100-
result.push_back(NamePair(origin_name, target_name));
101-
return pair;
102-
}
103-
}
104-
};
105-
106-
10730
inline void setAlterCommandColumn(Logger * log, AlterCommand & command, const ColumnInfo & column_info)
10831
{
10932
command.column_name = column_info.name;
@@ -152,7 +75,8 @@ inline std::vector<AlterCommands> detectSchemaChanges(Logger * log, const TableI
15275
}
15376

15477
{
155-
std::map<String, String> rename_map;
78+
using Resolver = CyclicRenameResolver<String, TmpColNameGenerator>;
79+
typename Resolver::NameMap rename_map;
15680
/// rename columns.
15781
for (const auto & orig_column_info : orig_table_info.columns)
15882
{
@@ -167,7 +91,7 @@ inline std::vector<AlterCommands> detectSchemaChanges(Logger * log, const TableI
16791
}
16892
}
16993

170-
auto rename_result = CyclicRenameResolver<String, TmpColNameGenerator>().resolve(rename_map);
94+
typename Resolver::NamePairs rename_result = Resolver().resolve(std::move(rename_map));
17195
for (const auto & rename_pair : rename_result)
17296
{
17397
AlterCommands rename_commands;
@@ -189,7 +113,7 @@ inline std::vector<AlterCommands> detectSchemaChanges(Logger * log, const TableI
189113
const auto & column_info
190114
= std::find_if(table_info.columns.begin(), table_info.columns.end(), [&](const ColumnInfo & column_info_) {
191115
if (column_info_.id == orig_column_info.id && column_info_.name != orig_column_info.name)
192-
LOG_ERROR(log, "detect column " << orig_column_info.name << " rename to " << column_info_.name);
116+
LOG_INFO(log, "detect column " << orig_column_info.name << " rename to " << column_info_.name);
193117

194118
return column_info_.id == orig_column_info.id
195119
&& (column_info_.tp != orig_column_info.tp || column_info_.hasNotNullFlag() != orig_column_info.hasNotNullFlag());
@@ -200,7 +124,7 @@ inline std::vector<AlterCommands> detectSchemaChanges(Logger * log, const TableI
200124
AlterCommand command;
201125
// Type changed column.
202126
command.type = AlterCommand::MODIFY_COLUMN;
203-
// Alter column with old name.
127+
// Alter column with new column info
204128
setAlterCommandColumn(log, command, *column_info);
205129
alter_commands.emplace_back(std::move(command));
206130
}
@@ -798,7 +722,7 @@ void SchemaBuilder<Getter>::alterAndRenameTables(std::vector<std::pair<TableInfo
798722
}
799723
}
800724

801-
auto result = Resolver().resolve(rename_map);
725+
typename Resolver::NamePairs result = Resolver().resolve(std::move(rename_map));
802726
for (const auto & rename_pair : result)
803727
{
804728
applyRenameTableImpl(rename_pair.first.first, rename_pair.second.first, rename_pair.first.second, rename_pair.second.second);

‎dbms/src/Storages/Transaction/SchemaBuilder.h

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <Interpreters/Context.h>
34
#include <Storages/StorageMergeTree.h>
45
#include <Storages/Transaction/SchemaGetter.h>
56

‎dbms/src/Storages/Transaction/tests/gtest_rename_resolver.cpp

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#include <test_utils/TiflashTestBasic.h>
2+
3+
#include <Storages/Transaction/SchemaBuilder-internal.h>
4+
#include <Storages/Transaction/SchemaBuilder.h>
5+
6+
namespace DB::tests
7+
{
8+
9+
TEST(CyclicRenameResolver_test, resolve_normal)
10+
{
11+
using Resolver = CyclicRenameResolver<String, TmpColNameGenerator>;
12+
std::map<String, String> rename_map;
13+
rename_map["a"] = "aa";
14+
rename_map["b"] = "bb";
15+
16+
typename Resolver::NamePairs rename_result = Resolver().resolve(std::move(rename_map));
17+
18+
ASSERT_EQ(rename_result.size(), 2UL);
19+
// a -> aa
20+
ASSERT_EQ(rename_result[0].first, "a");
21+
ASSERT_EQ(rename_result[0].second, "aa");
22+
// b -> bb
23+
ASSERT_EQ(rename_result[1].first, "b");
24+
ASSERT_EQ(rename_result[1].second, "bb");
25+
}
26+
27+
TEST(CyclicRenameResolver_test, resolve_linked)
28+
{
29+
using Resolver = CyclicRenameResolver<String, TmpColNameGenerator>;
30+
std::map<String, String> rename_map;
31+
rename_map["a"] = "c";
32+
rename_map["b"] = "a";
33+
34+
typename Resolver::NamePairs rename_result = Resolver().resolve(std::move(rename_map));
35+
36+
ASSERT_EQ(rename_result.size(), 2UL);
37+
// a -> c
38+
ASSERT_EQ(rename_result[0].first, "a");
39+
ASSERT_EQ(rename_result[0].second, "c");
40+
// b -> a
41+
ASSERT_EQ(rename_result[1].first, "b");
42+
ASSERT_EQ(rename_result[1].second, "a");
43+
}
44+
45+
TEST(CyclicRenameResolver_test, resolve_simple_cycle)
46+
{
47+
using Resolver = CyclicRenameResolver<String, TmpColNameGenerator>;
48+
std::map<String, String> rename_map;
49+
rename_map["a"] = "b";
50+
rename_map["b"] = "a";
51+
52+
typename Resolver::NamePairs rename_result = Resolver().resolve(std::move(rename_map));
53+
54+
TmpColNameGenerator generator;
55+
56+
ASSERT_EQ(rename_result.size(), 3UL);
57+
// a -> tmp_a
58+
ASSERT_EQ(rename_result[0].first, "a");
59+
ASSERT_EQ(rename_result[0].second, generator("a"));
60+
// b -> a
61+
ASSERT_EQ(rename_result[1].first, "b");
62+
ASSERT_EQ(rename_result[1].second, "a");
63+
// tmp_a -> b
64+
ASSERT_EQ(rename_result[2].first, generator("a"));
65+
ASSERT_EQ(rename_result[2].second, "b");
66+
}
67+
68+
TEST(CyclicRenameResolver_test, resolve_id_simple_cycle)
69+
{
70+
using Resolver = CyclicRenameResolver<ColumnNameWithID, TmpColNameWithIDGenerator>;
71+
std::map<ColumnNameWithID, ColumnNameWithID> rename_map;
72+
rename_map[ColumnNameWithID{"a", 1}] = ColumnNameWithID{"b", 1};
73+
rename_map[ColumnNameWithID{"b", 2}] = ColumnNameWithID{"a", 2};
74+
75+
typename Resolver::NamePairs rename_result = Resolver().resolve(std::move(rename_map));
76+
77+
TmpColNameWithIDGenerator generator;
78+
79+
ASSERT_EQ(rename_result.size(), 3UL);
80+
// a -> tmp_a
81+
ASSERT_EQ(rename_result[0].first, ColumnNameWithID("a", 1L));
82+
ASSERT_EQ(rename_result[0].second, generator(ColumnNameWithID{"a", 1}));
83+
// b -> a
84+
ASSERT_EQ(rename_result[1].first, ColumnNameWithID("b", 2L));
85+
ASSERT_EQ(rename_result[1].second, ColumnNameWithID("a", 2L));
86+
// tmp_a -> b
87+
ASSERT_EQ(rename_result[2].first, generator(ColumnNameWithID{"a", 1}));
88+
ASSERT_EQ(rename_result[2].second, ColumnNameWithID("b", 1));
89+
}
90+
91+
} // namespace DB

‎tests/mutable-test/txn_schema/rename_column.test

+27
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,30 @@
155155
=> DBGInvoke __drop_tidb_table(default, test)
156156
=> drop table if exists default.test
157157
=> DBGInvoke __refresh_schemas()
158+
159+
## test for partial-linked rename
160+
=> DBGInvoke __mock_tidb_table(default, test, 'a String, b Int8')
161+
=> DBGInvoke __refresh_schemas()
162+
=> DBGInvoke __put_region(4, 0, 100, default, test)
163+
=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test', 1)
164+
=> DBGInvoke __raft_insert_row(default, test, 4, 51, 'test', 2)
165+
=> DBGInvoke __try_flush_region(4)
166+
=> select a, b from default.test order by _tidb_rowid
167+
┌─a────┬─b─┐
168+
│ test │ 1 │
169+
│ test │ 2 │
170+
└──────┴───┘
171+
172+
# rename a -> c, and b -> a
173+
=> DBGInvoke __rename_column_in_tidb_table(default, test, a, c)
174+
=> DBGInvoke __rename_column_in_tidb_table(default, test, b, a)
175+
=> DBGInvoke __refresh_schemas()
176+
=> select a, c from default.test order by _tidb_rowid
177+
┌─a─┬─c────┐
178+
│ 1 │ test │
179+
│ 2 │ test │
180+
└───┴──────┘
181+
182+
=> DBGInvoke __drop_tidb_table(default, test)
183+
=> drop table if exists default.test
184+
=> DBGInvoke __refresh_schemas()

0 commit comments

Comments
 (0)