From ec4caa3c7794e8f7a1d7412712a1b98af7c53282 Mon Sep 17 00:00:00 2001 From: haohang Date: Sat, 17 Aug 2024 03:35:39 -0700 Subject: [PATCH] support force drop big stream --- src/Core/Settings.h | 1 + src/Interpreters/Context.cpp | 10 +++++++--- src/Storages/StorageMergeTree.cpp | 4 ++-- .../0_stateless/99006_force_drop.reference | 0 tests/queries_ported/0_stateless/99006_force_drop.sql | 5 +++++ 5 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 tests/queries_ported/0_stateless/99006_force_drop.reference create mode 100644 tests/queries_ported/0_stateless/99006_force_drop.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 10e1dbd8f77..e2fb8e4b7c6 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -828,6 +828,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(String, replay_time_column, "_tp_append_time", "user specified replay time column, default column is _tp_append_time", 0) \ M(UInt64, max_events, 0, "Total events to generate for random stream", 0) \ M(Float, eps, -1., "control the random stream eps in query time, defalut value is -1, if it is 0 means no limit.", 0) \ + M(Bool, force_drop_big_stream, false, "When enabled, the query can directly force drop big stream greater than configured `max_[stream/partition]_size_to_drop`", 0) \ // End of GLOBAL_SETTINGS #define CONFIGURABLE_GLOBAL_SETTINGS(M) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bffeab7c5e9..efab4b90f7a 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2803,8 +2803,10 @@ const StorageS3Settings & Context::getStorageS3Settings() const void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const { - if (!max_size_to_drop || size <= max_size_to_drop) + /// proton: starts. add setting `force_drop_big_stream` + if (!max_size_to_drop || size <= max_size_to_drop || getSettingsRef().force_drop_big_stream) return; + /// proton: ends. fs::path force_file(getFlagsPath() + "force_drop_table"); bool force_file_exists = fs::exists(force_file); @@ -2827,11 +2829,13 @@ void Context::checkCanBeDropped(const String & database, const String & table, c String max_size_to_drop_str = formatReadableSizeWithDecimalSuffix(max_size_to_drop); throw Exception(ErrorCodes::STREAM_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT, "Stream or Partition in {}.{} was not dropped.\nReason:\n" - "1. Size ({}) is greater than max_[table/partition]_size_to_drop ({})\n" + "1. Size ({}) is greater than max_[stream/partition]_size_to_drop ({})\n" "2. File '{}' intended to force DROP {}\n" + "3. Setting 'force_drop_big_stream' is not set\n" "How to fix this:\n" "1. Either increase (or set to zero) max_[stream/partition]_size_to_drop in server config\n" - "2. Either create forcing file {} and make sure that proton has write permission for it.\n" + "2. Either create forcing file {} and make sure that timeplus DBMS has write permission for it.\n" + "3. Either add setting 'force_drop_big_stream=true'\n" "Example:\nsudo touch '{}' && sudo chmod 666 '{}'", backQuoteIfNeed(database), backQuoteIfNeed(table), size_str, max_size_to_drop_str, diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index db05f4cab60..58d1d9d1cbc 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -317,10 +317,10 @@ StorageMergeTree::write(const ASTPtr & /*query*/, const StorageMetadataPtr & met void StorageMergeTree::checkTableCanBeDropped(ContextPtr context_) const { IStorage::checkTableCanBeDropped(context_); -/// proton: ends. auto table_id = getStorageID(); - getContext()->checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes()); + context_->checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes()); +/// proton: ends. } void StorageMergeTree::drop() diff --git a/tests/queries_ported/0_stateless/99006_force_drop.reference b/tests/queries_ported/0_stateless/99006_force_drop.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries_ported/0_stateless/99006_force_drop.sql b/tests/queries_ported/0_stateless/99006_force_drop.sql new file mode 100644 index 00000000000..ee8ffe436d1 --- /dev/null +++ b/tests/queries_ported/0_stateless/99006_force_drop.sql @@ -0,0 +1,5 @@ +CREATE STREAM 99006_v(id int); + +INSERT INTO 99006_v(id) values(3); + +DROP STREAM 99006_v settings force_drop_big_stream = true;