-
Notifications
You must be signed in to change notification settings - Fork 411
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
18 changed files
with
206 additions
and
68 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,41 +1,76 @@ | ||
| #include <Storages/MergeTree/DiskSpaceMonitor.h> | ||
| #include <Storages/MergeTree/MergeTreeData.h> | ||
| #include <Storages/MergeTree/MergeTreePartInfo.h> | ||
| #include <Storages/PartPathSelector.h> | ||
| #include <common/likely.h> | ||
| #include <map> | ||
|
|
||
| namespace DB | ||
| { | ||
| const String PartPathSelector::getPathForPart(MergeTreeData & data, const String & part_name) const | ||
| const String PartPathSelector::getPathForPart( | ||
| MergeTreeData & data, const String & part_name, const MergeTreePartInfo & info, size_t part_size) const | ||
| { | ||
| if (all_path.size() == 1) | ||
| // test whether this part can be put on fast path and return the path if it can | ||
| if (hasFastPath()) | ||
| { | ||
| return all_path[0]; | ||
| if (info.level == 0) | ||
| { | ||
| return normal_and_fast_path[getRandomFastPathIndex()]; | ||
| } | ||
| size_t max_available_space = DiskSpaceMonitor::getUnreservedFreeSpace(normal_and_fast_path[fast_path_start_index]); | ||
| size_t path_index = fast_path_start_index; | ||
| for (size_t i = fast_path_start_index + 1; i < normal_and_fast_path.size(); i++) | ||
| { | ||
| size_t s = DiskSpaceMonitor::getUnreservedFreeSpace(normal_and_fast_path[i]); | ||
| if (s > max_available_space) | ||
| { | ||
| max_available_space = s; | ||
| path_index = i; | ||
| } | ||
| } | ||
|
|
||
| if (max_available_space >= settings.min_space_reserved_for_level_zero_parts | ||
| && part_size <= settings.part_other_than_level_zero_max_ratio * max_available_space) | ||
| { | ||
| return normal_and_fast_path[path_index]; | ||
| } | ||
| } | ||
|
|
||
| // there is only one normal path, just return it | ||
| if (fast_path_start_index == 1) | ||
| { | ||
| return normal_and_fast_path[0]; | ||
| } | ||
| std::unordered_map<String, UInt64> path_size_map; | ||
| for (const auto & path : all_path) | ||
| if (info.level == 0) | ||
| { | ||
| path_size_map.emplace(path, 0); | ||
| return normal_and_fast_path[getRandomNormalPathIndex()]; | ||
| } | ||
| // find the normal path with least size of parts of this table | ||
| std::vector<UInt64> path_parts_size; | ||
| path_parts_size.resize(fast_path_start_index, 0); | ||
| for (const auto & part : data.getDataPartsVector()) | ||
| { | ||
| if (unlikely(path_size_map.find(part->full_path_prefix) == path_size_map.end())) | ||
| if (unlikely(normal_path_to_index_map.find(part->full_path_prefix) == normal_path_to_index_map.end())) | ||
| { | ||
| throw Exception("Part " + part->relative_path + " got unexpected path " + part->full_path_prefix, ErrorCodes::LOGICAL_ERROR); | ||
| } | ||
| path_size_map[part->full_path_prefix] += part->bytes_on_disk; | ||
| path_parts_size[normal_path_to_index_map.at(part->full_path_prefix)] += part->bytes_on_disk; | ||
| } | ||
| String result = all_path[0]; | ||
| UInt64 parts_size = path_size_map[result]; | ||
| for (const auto & element : path_size_map) | ||
| size_t result_path_index = 0; | ||
| UInt64 parts_size_on_result_path = path_parts_size[0]; | ||
| for (size_t i = 1; i < fast_path_start_index; i++) | ||
| { | ||
| if (element.second < parts_size) | ||
| if (path_parts_size[i] < parts_size_on_result_path) | ||
| { | ||
| result = element.first; | ||
| parts_size = element.second; | ||
| result_path_index = i; | ||
| parts_size_on_result_path = path_parts_size[i]; | ||
| } | ||
| LOG_DEBUG(log, "Path " << element.first << " size is " << element.second << " bytes."); | ||
| LOG_TRACE(log, "Path " << normal_and_fast_path[i] << " size is " << path_parts_size[i] << " bytes."); | ||
| } | ||
| LOG_TRACE(log, | ||
| "database: " << data.getDatabaseName() << " table: " << data.getTableName() << " part name: " << part_name | ||
| << " path: " << normal_and_fast_path[result_path_index]); | ||
|
|
||
| LOG_DEBUG(log, "database: " << data.getDatabaseName() << " table: " << data.getTableName() << " part name: " << part_name << " path: " << result); | ||
| return result; | ||
| return normal_and_fast_path[result_path_index]; | ||
| } | ||
| } // namespace DB |
Oops, something went wrong.