From a3bb1b4c5d583163b1dfdec44ed53941c45a123d Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Wed, 27 Nov 2024 22:05:35 +0800 Subject: [PATCH 1/5] [improve](mow) merge and remove old version of delete bitmap when cumulative compaction is done on local mode --- be/src/cloud/cloud_cumulative_compaction.cpp | 28 ++------ .../action/delete_bitmap_action.cpp} | 26 ++++--- .../action/delete_bitmap_action.h} | 13 ++-- be/src/olap/compaction.cpp | 26 +++++++ be/src/olap/compaction.h | 6 ++ be/src/olap/cumulative_compaction.cpp | 34 +++++++++ be/src/olap/cumulative_compaction.h | 2 + be/src/olap/tablet.cpp | 10 +++ be/src/olap/tablet_meta.cpp | 2 +- be/src/service/http_service.cpp | 21 ++++-- ...on_remove_old_version_delete_bitmap.groovy | 71 +++++++++---------- 11 files changed, 154 insertions(+), 85 deletions(-) rename be/src/{cloud/cloud_delete_bitmap_action.cpp => http/action/delete_bitmap_action.cpp} (88%) rename be/src/{cloud/cloud_delete_bitmap_action.h => http/action/delete_bitmap_action.h} (79%) diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index f63054563aa18a..6b4dbe360da651 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -383,28 +383,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { } std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); if (!pre_rowsets.empty()) { - auto pre_max_version = _output_rowset->version().second; - DeleteBitmapPtr new_delete_bitmap = - std::make_shared(_tablet->tablet_meta()->tablet_id()); std::vector> to_remove_vec; - for (auto& rowset : pre_rowsets) { - if (rowset->rowset_meta()->total_disk_size() == 0) { - continue; - } - for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { - rowset->rowset_id().to_string(); - DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; - DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; - auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( - {rowset->rowset_id(), seg_id, pre_max_version}); - to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); - if (d->isEmpty()) { - continue; - } - new_delete_bitmap->set(end, *d); - } - } + DeleteBitmapPtr new_delete_bitmap = nullptr; + agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); if (!new_delete_bitmap->empty()) { // store agg delete bitmap DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed", @@ -424,9 +406,9 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { } _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), to_remove_vec); - DBUG_EXECUTE_IF( - "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", - { static_cast(_tablet.get())->delete_expired_stale_rowsets(); }); + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { + static_cast(_tablet.get())->delete_expired_stale_rowsets(); + }); } } return Status::OK(); diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp similarity index 88% rename from be/src/cloud/cloud_delete_bitmap_action.cpp rename to be/src/http/action/delete_bitmap_action.cpp index 3d834bfe7b373c..54652bad06c64c 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ b/be/src/http/action/delete_bitmap_action.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "cloud_delete_bitmap_action.h" +#include "delete_bitmap_action.h" #include #include @@ -34,8 +34,10 @@ #include #include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "common/logging.h" #include "common/status.h" #include "gutil/strings/substitute.h" @@ -44,7 +46,6 @@ #include "http/http_request.h" #include "http/http_status.h" #include "olap/olap_define.h" -#include "olap/storage_engine.h" #include "olap/tablet_manager.h" #include "util/doris_metrics.h" #include "util/stopwatch.hpp" @@ -59,10 +60,9 @@ constexpr std::string_view HEADER_JSON = "application/json"; } // namespace -CloudDeleteBitmapAction::CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, - TPrivilegeHier::type hier, - TPrivilegeType::type ptype) +DeleteBitmapAction::DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, + BaseStorageEngine& engine, TPrivilegeHier::type hier, + TPrivilegeType::type ptype) : HttpHandlerWithAuth(exec_env, hier, ptype), _engine(engine), _delete_bitmap_action_type(ctype) {} @@ -80,7 +80,7 @@ static Status _check_param(HttpRequest* req, uint64_t* tablet_id) { return Status::OK(); } -Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, +Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains @@ -89,11 +89,15 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque return Status::InternalError("check param failed: missing tablet_id"); } - CloudTabletSPtr tablet = DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id)); + BaseTabletSPtr tablet = nullptr; + if (config::is_cloud_mode()) { + tablet = DORIS_TRY(_engine.to_cloud().tablet_mgr().get_tablet(tablet_id)); + } else { + tablet = _engine.to_local().tablet_manager()->get_tablet(tablet_id); + } if (tablet == nullptr) { return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); } - auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); auto size = tablet->tablet_meta()->delete_bitmap().get_size(); @@ -115,7 +119,7 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque return Status::OK(); } -Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, +Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains @@ -157,7 +161,7 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* return Status::OK(); } -void CloudDeleteBitmapAction::handle(HttpRequest* req) { +void DeleteBitmapAction::handle(HttpRequest* req) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data()); if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) { std::string json_result; diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/http/action/delete_bitmap_action.h similarity index 79% rename from be/src/cloud/cloud_delete_bitmap_action.h rename to be/src/http/action/delete_bitmap_action.h index ce507ee9991757..284e8dbcf5705b 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.h +++ b/be/src/http/action/delete_bitmap_action.h @@ -21,9 +21,9 @@ #include -#include "cloud/cloud_storage_engine.h" #include "common/status.h" #include "http/http_handler_with_auth.h" +#include "olap/storage_engine.h" #include "olap/tablet.h" namespace doris { @@ -35,13 +35,12 @@ class ExecEnv; enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 }; /// This action is used for viewing the delete bitmap status -class CloudDeleteBitmapAction : public HttpHandlerWithAuth { +class DeleteBitmapAction : public HttpHandlerWithAuth { public: - CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, TPrivilegeHier::type hier, - TPrivilegeType::type ptype); + DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, BaseStorageEngine& engine, + TPrivilegeHier::type hier, TPrivilegeType::type ptype); - ~CloudDeleteBitmapAction() override = default; + ~DeleteBitmapAction() override = default; void handle(HttpRequest* req) override; @@ -50,7 +49,7 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth { Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result); private: - CloudStorageEngine& _engine; + BaseStorageEngine& _engine; DeleteBitmapActionType _delete_bitmap_action_type; }; #include "common/compile_check_end.h" diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 8c45c20f799427..1e5b1bc5722ca9 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -927,6 +927,32 @@ Status CloudCompactionMixin::update_delete_bitmap() { << ", rowsets: " << _input_rowsets.size() << ", cost: " << watch.get_elapse_time_us() << "(us)"; return Status::OK(); + +void Compaction::agg_and_remove_old_version_delete_bitmap( + std::vector& pre_rowsets, + std::vector>& + to_remove_vec, + DeleteBitmapPtr& new_delete_bitmap) { + // agg previously rowset old version delete bitmap + auto pre_max_version = _output_rowset->version().second; + new_delete_bitmap = std::make_shared(_tablet->tablet_meta()->tablet_id()); + for (auto& rowset : pre_rowsets) { + if (rowset->rowset_meta()->total_disk_size() == 0) { + continue; + } + for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { + rowset->rowset_id().to_string(); + DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; + DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; + auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( + {rowset->rowset_id(), seg_id, pre_max_version}); + to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); + if (d->isEmpty()) { + continue; + } + new_delete_bitmap->set(end, *d); + } + } } Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) { diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 7f92a6c5f4d7dc..d8e6365ce8d468 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -86,6 +86,12 @@ class Compaction { virtual Status update_delete_bitmap() = 0; + void agg_and_remove_old_version_delete_bitmap( + std::vector& pre_rowsets, + std::vector>& + to_remove_vec, + DeleteBitmapPtr& new_delete_bitmap); + // the root tracker for this compaction std::shared_ptr _mem_tracker; diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 2dfd30fb86ed9a..6a4733a26a8d09 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -124,6 +124,10 @@ Status CumulativeCompaction::execute_compact() { _output_rowset->num_segments() == 0)) { tablet()->set_last_cumu_compaction_success_time(UnixMillis()); } + if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { + process_old_version_delete_bitmap(); + } DorisMetrics::instance()->cumulative_compaction_deltas_total->increment(_input_rowsets.size()); DorisMetrics::instance()->cumulative_compaction_bytes_total->increment( _input_rowsets_total_size); @@ -220,4 +224,34 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { return Status::OK(); } +void CumulativeCompaction::process_old_version_delete_bitmap() { + std::vector pre_rowsets {}; + for (const auto& it : tablet()->rowset_map()) { + if (it.first.second < _input_rowsets.front()->start_version()) { + pre_rowsets.emplace_back(it.second); + } + } + std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); + if (!pre_rowsets.empty()) { + std::vector> + to_remove_vec; + DeleteBitmapPtr new_delete_bitmap = nullptr; + agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); + if (!new_delete_bitmap->empty()) { + // store agg delete bitmap + Version version(_input_rowsets.front()->start_version(), + _input_rowsets.back()->end_version()); + for (auto it = new_delete_bitmap->delete_bitmap.begin(); + it != new_delete_bitmap->delete_bitmap.end(); it++) { + _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); + } + _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), + to_remove_vec); + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { + static_cast(_tablet.get())->delete_expired_stale_rowset(); + }); + } + } +} + } // namespace doris diff --git a/be/src/olap/cumulative_compaction.h b/be/src/olap/cumulative_compaction.h index 276e3b3490311c..0d8c684740275a 100644 --- a/be/src/olap/cumulative_compaction.h +++ b/be/src/olap/cumulative_compaction.h @@ -47,6 +47,8 @@ class CumulativeCompaction final : public CompactionMixin { void find_longest_consecutive_version(std::vector* rowsets, std::vector* missing_version); + void process_old_version_delete_bitmap(); + Version _last_delete_version {-1, -1}; }; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 379fb6eec3cb43..ede28c7f1dbda8 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -817,10 +817,13 @@ void Tablet::delete_expired_stale_rowset() { auto old_meta_size = _tablet_meta->all_stale_rs_metas().size(); // do delete operation + std::vector version_to_delete; auto to_delete_iter = stale_version_path_map.begin(); while (to_delete_iter != stale_version_path_map.end()) { std::vector& to_delete_version = to_delete_iter->second->timestamped_versions(); + int start_version = -1; + int end_version = -1; for (auto& timestampedVersion : to_delete_version) { auto it = _stale_rs_version_map.find(timestampedVersion->version()); if (it != _stale_rs_version_map.end()) { @@ -841,10 +844,17 @@ void Tablet::delete_expired_stale_rowset() { << timestampedVersion->version().second << "] not find in stale rs version map"; } + if (start_version < 0) { + start_version = timestampedVersion->version().first; + } + end_version = timestampedVersion->version().second; _delete_stale_rowset_by_version(timestampedVersion->version()); } + Version version(start_version, end_version); + version_to_delete.emplace_back(version.to_string()); to_delete_iter++; } + _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete); bool reconstructed = _reconstruct_version_tracker_if_necessary(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3247f34656fb5d..60f6b59ca26b29 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1219,7 +1219,7 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vectorstorage_engine().to_cloud(); diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 912f9f5ff403e7..548d7d1ad0e0b9 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -25,7 +25,6 @@ #include #include "cloud/cloud_compaction_action.h" -#include "cloud/cloud_delete_bitmap_action.h" #include "cloud/config.h" #include "cloud/injection_point_action.h" #include "common/config.h" @@ -42,6 +41,7 @@ #include "http/action/compaction_score_action.h" #include "http/action/config_action.h" #include "http/action/debug_point_action.h" +#include "http/action/delete_bitmap_action.h" #include "http/action/download_action.h" #include "http/action/download_binlog_action.h" #include "http/action/file_cache_action.h" @@ -378,6 +378,13 @@ void HttpService::register_local_handler(StorageEngine& engine) { _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); + + DeleteBitmapAction* count_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_INFO, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", + count_delete_bitmap_action); + CheckTabletSegmentAction* check_tablet_segment_action = _pool.add(new CheckTabletSegmentAction( _env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::POST, "/api/check_tablet_segment_lost", @@ -426,14 +433,14 @@ void HttpService::register_cloud_handler(CloudStorageEngine& engine) { TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); - CloudDeleteBitmapAction* count_local_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_local_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", count_local_delete_bitmap_action); - CloudDeleteBitmapAction* count_ms_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_ms_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_ms", count_ms_delete_bitmap_action); #ifdef ENABLE_INJECTION_POINT diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy index a36cb4579ca487..ec8888d287c9b4 100644 --- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy @@ -18,9 +18,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { - if (!isCloudMode()) { - return - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] def backendId_to_params = [string: [:]] @@ -189,7 +186,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { set_be_param("tablet_rowset_stale_sweep_time_sec", "0") try { - GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") // 1. test normal sql "sync" sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """ @@ -319,37 +316,39 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { qt_sql "select * from ${testTable} order by plan_id" GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId - - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_cardinality == 12) - assertTrue(ms_delete_bitmap_cardinality == 12) - - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - - // update fail, local delete_bitmap_count will not change - Thread.sleep(1000) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) + if (isCloudMode()) { + for (def tablet in tablets) { + String tablet_id = tablet.TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + String trigger_backend_id = tablet.BackendId + + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + assertTrue(local_delete_bitmap_count == 6) + assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) + + local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_cardinality == 12) + assertTrue(ms_delete_bitmap_cardinality == 12) + + getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], + "cumulative", tablet_id).contains("Success")); + waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); + + // update fail, local delete_bitmap_count will not change + Thread.sleep(1000) + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + assertTrue(local_delete_bitmap_count == 6) + } } qt_sql "select * from ${testTable} order by plan_id" @@ -372,7 +371,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { } finally { reset_be_param("compaction_promotion_version_count") reset_be_param("tablet_rowset_stale_sweep_time_sec") - GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") + GetDebugPoint().disableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") } From 1763a31633a58c54726ed0d4ca8e46fdf20ed3b3 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Fri, 29 Nov 2024 11:42:33 +0800 Subject: [PATCH 2/5] edit --- be/src/http/action/delete_bitmap_action.cpp | 10 +-- be/src/service/http_service.cpp | 2 +- ...on_remove_old_version_delete_bitmap.groovy | 63 ++++++++++--------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/be/src/http/action/delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp index 54652bad06c64c..2fa0a73c2f338c 100644 --- a/be/src/http/action/delete_bitmap_action.cpp +++ b/be/src/http/action/delete_bitmap_action.cpp @@ -81,7 +81,7 @@ static Status _check_param(HttpRequest* req, uint64_t* tablet_id) { } Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { + std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); @@ -120,7 +120,7 @@ Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* r } Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { + std::string* json_result) { uint64_t tablet_id = 0; // check & retrieve tablet_id from req if it contains RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); @@ -128,14 +128,14 @@ Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, return Status::InternalError("check param failed: missing tablet_id"); } TabletMetaSharedPtr tablet_meta; - auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); + auto st = _engine.to_cloud().meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); if (!st.ok()) { LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id << ", st=" << st.to_string(); return st; } - auto tablet = std::make_shared(_engine, std::move(tablet_meta)); - st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); + auto tablet = std::make_shared(_engine.to_cloud(), std::move(tablet_meta)); + st = _engine.to_cloud().meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); if (!st.ok()) { LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st; return st; diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 548d7d1ad0e0b9..7f2d84dc2a8e11 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -380,7 +380,7 @@ void HttpService::register_local_handler(StorageEngine& engine) { run_status_compaction_action); DeleteBitmapAction* count_delete_bitmap_action = - _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_INFO, _env, engine, + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", count_delete_bitmap_action); diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy index ec8888d287c9b4..e14bdbbb2c6505 100644 --- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy @@ -160,7 +160,7 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { def timeout = 10000 sql """ DROP TABLE IF EXISTS ${testTable}""" def testTableDDL = """ - create table ${testTable} + create table ${testTable} ( `plan_id` bigint(20) NOT NULL, `target_id` int(20) NOT NULL, @@ -216,18 +216,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { // before compaction, delete_bitmap_count is (rowsets num - 1) local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 7) assertTrue(local_delete_bitmap_cardinality == 7) - assertTrue(local_delete_bitmap_cardinality == ms_delete_bitmap_cardinality) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 7) + assertTrue(ms_delete_bitmap_cardinality == 7) + } assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], @@ -262,18 +264,20 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { // before compaction, local delete_bitmap_count is (total rowsets num - 1), ms delete_bitmap_count is new rowset num String trigger_backend_id = tablet.BackendId local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 12) - assertTrue(ms_delete_bitmap_count == 5) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 12) assertTrue(local_delete_bitmap_cardinality == 17) - assertTrue(ms_delete_bitmap_cardinality == 10) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 5) + assertTrue(ms_delete_bitmap_cardinality == 10) + } getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], @@ -282,20 +286,21 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); Thread.sleep(1000) - // after compaction, delete_bitmap_count is 1 + // after compaction, delete_bitmap_count is 1, cardinality is 2, check it local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 1) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 1) assertTrue(local_delete_bitmap_cardinality == 2) - assertTrue(ms_delete_bitmap_cardinality == 2) + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 1) + assertTrue(ms_delete_bitmap_cardinality == 2) + } } qt_sql "select * from ${testTable} order by plan_id" From 99c8b0c529cb402f479de27dee8d96b51a730eab Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Mon, 2 Dec 2024 11:52:47 +0800 Subject: [PATCH 3/5] edit --- be/src/olap/compaction.cpp | 36 +++++++++++++++++++++++ be/src/olap/compaction.h | 2 ++ be/src/olap/cumulative_compaction.cpp | 34 --------------------- be/src/olap/cumulative_compaction.h | 2 -- be/src/olap/rowset/beta_rowset_writer.cpp | 3 +- be/src/olap/tablet.cpp | 4 +-- be/src/olap/tablet_meta.cpp | 16 ++++++++-- 7 files changed, 55 insertions(+), 42 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 1e5b1bc5722ca9..f1e7502877595a 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1129,6 +1129,12 @@ Status CompactionMixin::modify_rowsets() { tablet()->delete_expired_stale_rowset(); } + if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && + _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { + process_old_version_delete_bitmap(); + } + int64_t cur_max_version = 0; { std::shared_lock rlock(_tablet->get_header_lock()); @@ -1147,6 +1153,36 @@ Status CompactionMixin::modify_rowsets() { return Status::OK(); } +void CompactionMixin::process_old_version_delete_bitmap() { + std::vector pre_rowsets {}; + for (const auto& it : tablet()->rowset_map()) { + if (it.first.second < _input_rowsets.front()->start_version()) { + pre_rowsets.emplace_back(it.second); + } + } + std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); + if (!pre_rowsets.empty()) { + std::vector> + to_remove_vec; + DeleteBitmapPtr new_delete_bitmap = nullptr; + agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); + if (!new_delete_bitmap->empty()) { + // store agg delete bitmap + Version version(_input_rowsets.front()->start_version(), + _input_rowsets.back()->end_version()); + for (auto it = new_delete_bitmap->delete_bitmap.begin(); + it != new_delete_bitmap->delete_bitmap.end(); it++) { + _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); + } + _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), + to_remove_vec); + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { + static_cast(_tablet.get())->delete_expired_stale_rowset(); + }); + } + } +} + bool CompactionMixin::_check_if_includes_input_rowsets( const RowsetIdUnorderedSet& commit_rowset_ids_set) const { std::vector commit_rowset_ids {}; diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index d8e6365ce8d468..ccabf7dadb4733 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -168,6 +168,8 @@ class CompactionMixin : public Compaction { Status do_compact_ordered_rowsets(); + void process_old_version_delete_bitmap(); + bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const; PendingRowsetGuard _pending_rs_guard; diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 6a4733a26a8d09..2dfd30fb86ed9a 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -124,10 +124,6 @@ Status CumulativeCompaction::execute_compact() { _output_rowset->num_segments() == 0)) { tablet()->set_last_cumu_compaction_success_time(UnixMillis()); } - if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { - process_old_version_delete_bitmap(); - } DorisMetrics::instance()->cumulative_compaction_deltas_total->increment(_input_rowsets.size()); DorisMetrics::instance()->cumulative_compaction_bytes_total->increment( _input_rowsets_total_size); @@ -224,34 +220,4 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { return Status::OK(); } -void CumulativeCompaction::process_old_version_delete_bitmap() { - std::vector pre_rowsets {}; - for (const auto& it : tablet()->rowset_map()) { - if (it.first.second < _input_rowsets.front()->start_version()) { - pre_rowsets.emplace_back(it.second); - } - } - std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); - if (!pre_rowsets.empty()) { - std::vector> - to_remove_vec; - DeleteBitmapPtr new_delete_bitmap = nullptr; - agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); - if (!new_delete_bitmap->empty()) { - // store agg delete bitmap - Version version(_input_rowsets.front()->start_version(), - _input_rowsets.back()->end_version()); - for (auto it = new_delete_bitmap->delete_bitmap.begin(); - it != new_delete_bitmap->delete_bitmap.end(); it++) { - _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); - } - _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), - to_remove_vec); - DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { - static_cast(_tablet.get())->delete_expired_stale_rowset(); - }); - } - } -} - } // namespace doris diff --git a/be/src/olap/cumulative_compaction.h b/be/src/olap/cumulative_compaction.h index 0d8c684740275a..276e3b3490311c 100644 --- a/be/src/olap/cumulative_compaction.h +++ b/be/src/olap/cumulative_compaction.h @@ -47,8 +47,6 @@ class CumulativeCompaction final : public CompactionMixin { void find_longest_consecutive_version(std::vector* rowsets, std::vector* missing_version); - void process_old_version_delete_bitmap(); - Version _last_delete_version {-1, -1}; }; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index ab5bc48db80b00..dc155efe0165bc 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -337,7 +337,8 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id() << ", rowset_ids: " << _context.mow_context->rowset_ids.size() << ", cur max_version: " << _context.mow_context->max_version - << ", transaction_id: " << _context.mow_context->txn_id + << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: " + << _context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count() << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows; return Status::OK(); } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index ede28c7f1dbda8..644ca9133eb885 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -822,8 +822,8 @@ void Tablet::delete_expired_stale_rowset() { while (to_delete_iter != stale_version_path_map.end()) { std::vector& to_delete_version = to_delete_iter->second->timestamped_versions(); - int start_version = -1; - int end_version = -1; + int64_t start_version = -1; + int64_t end_version = -1; for (auto& timestampedVersion : to_delete_version) { auto it = _stale_rs_version_map.find(timestampedVersion->version()); if (it != _stale_rs_version_map.end()) { diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 60f6b59ca26b29..43b0d5d8bd0ae0 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1091,7 +1091,9 @@ uint64_t DeleteBitmap::cardinality() const { std::shared_lock l(lock); uint64_t res = 0; for (auto entry : delete_bitmap) { - res += entry.second.cardinality(); + if (std::get<1>(entry.first) != DeleteBitmap::INVALID_SEGMENT_ID) { + res += entry.second.cardinality(); + } } return res; } @@ -1100,7 +1102,9 @@ uint64_t DeleteBitmap::get_size() const { std::shared_lock l(lock); uint64_t charge = 0; for (auto& [k, v] : delete_bitmap) { - charge += v.getSizeInBytes(); + if (std::get<1>(k) != DeleteBitmap::INVALID_SEGMENT_ID) { + charge += v.getSizeInBytes(); + } } return charge; } @@ -1232,7 +1236,13 @@ void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) { + count++; + } + } + return count; } // We cannot just copy the underlying memory to construct a string From c7b355a8c79b6b4ea42ae7e6092627d4d3f23d32 Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Wed, 4 Dec 2024 17:41:23 +0800 Subject: [PATCH 4/5] edit --- be/src/olap/compaction.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index f1e7502877595a..253f42ff8b6b94 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1129,7 +1129,8 @@ Status CompactionMixin::modify_rowsets() { tablet()->delete_expired_stale_rowset(); } - if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && + if (config::enable_delete_bitmap_merge_on_compaction && + compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && _tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { process_old_version_delete_bitmap(); From f3ee6b036ebd9b5a8d656a6e4f38984a24aa398a Mon Sep 17 00:00:00 2001 From: huanghaibin Date: Wed, 4 Dec 2024 19:13:25 +0800 Subject: [PATCH 5/5] edit --- be/src/olap/compaction.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 253f42ff8b6b94..7b2bb5d4ee3cfb 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -927,6 +927,7 @@ Status CloudCompactionMixin::update_delete_bitmap() { << ", rowsets: " << _input_rowsets.size() << ", cost: " << watch.get_elapse_time_us() << "(us)"; return Status::OK(); +} void Compaction::agg_and_remove_old_version_delete_bitmap( std::vector& pre_rowsets,