Skip to content

Commit b58570a

Browse files
xinyiZzzxinyiZzz
authored
[fix](memory) Refactor MemCounter (#40542)
1. Add a new class MemCounter. 2. MemTracker and MemTrackerLimiter no longer have inheritance and dependencies. 3. MemTrackerLimiter is used to count process memory, the BE web page `/mem_tracker` also only displays MemTrackerLimiter. 4. MemTracker is used to count the memory of operators and some data structures. It is not counted in the process memory and is used for logic control and profile. In addition, it seems that the crash is caused by memory abnormality, not sure whether this PR can fix the problem, but this will help locate the problem and more elegant. ``` ==6641==ERROR: AddressSanitizer: heap-use-after-free on address 0x603000980e70 at pc 0x559be21880be bp 0x7fd7cfc75070 sp 0x7fd7cfc75068 READ of size 8 at 0x603000980e70 thread T1453 (memory_maintena) #0 0x559be21880bd in std::__atomic_base<long>::load(std::memory_order) const /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/atomic_base.h:481:9 #1 0x559be21880bd in doris::MemTracker::MemCounter::current_value() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:139:63 #2 0x559be21880bd in doris::MemTracker::consumption() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:165:56 #3 0x559be3985133 in doris::MemTrackerLimiter::refresh_global_counter() /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker_limiter.cpp:245:59 #4 0x559bdfcd52d5 in doris::Daemon::memory_maintenance_thread() /home/zcp/repo_center/doris_master/doris/be/src/common/daemon.cpp:239:13 #5 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5 #6 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8 #7 0x7fdf5647584f misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 0x603000980e74 is located 0 bytes after 20-byte region [0x603000980e60,0x603000980e74) 04:19:48  freed by thread T1491 (Scan_normal [wo) here: 04:19:48  #0 0x559bdfb0dd9d in operator delete(void*) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x33546d9d) (BuildId: f170e92ad3c55512) 04:19:48  #1 0x559bdfb1d42b in __gnu_cxx::new_allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:139:2 04:19:48  #2 0x559bdfb1d42b in std::allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:187:27 04:19:48  #3 0x559bdfb1d42b in std::allocator_traits<std::allocator<char>>::deallocate(std::allocator<char>&, char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:492:13 04:19:48  #4 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_destroy(unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:237:9 04:19:48  #5 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_dispose() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:232:4 04:19:48  #6 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::~basic_string() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:658:9 04:19:48  #7 0x559bdfb22501 in void std::destroy_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:88:15 04:19:48  #8 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:138:7 04:19:48  #9 0x559bdfb22501 in void std::_Destroy_aux<false>::__destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:152:6 04:19:48  #10 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:184:7 04:19:48  #11 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:746:7 04:19:48  #12 0x559bdfb22501 in std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::~vector() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_vector.h:680:2 04:19:48  #13 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9867:1 04:19:48  #14 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9866:36 04:19:48  #15 0x559be0047ee6 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:372:5 04:19:48  #16 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13 04:19:48  #17 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28 04:19:48  #18 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18 04:19:48  #19 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18 04:19:48  #20 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5 04:19:48  #21 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32 04:19:48  #22 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5 04:19:48  #23 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21 04:19:48  #24 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31 04:19:48  #25 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14 04:19:48  #26 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2 04:19:48  #27 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9 04:19:48  #28 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24 04:19:48  #29 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5 04:19:48  #30 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8 04:19:48  04:19:48  previously allocated by thread T1491 (Scan_normal [wo) here: 04:19:48  #0 0x559bdfb0d53d in operator new(unsigned long) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x3354653d) (BuildId: f170e92ad3c55512) 04:19:48  #1 0x559bdfb33f74 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:219:14 04:19:48  #2 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct_aux<char*>(char*, char*, std::__false_type) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:247:11 04:19:48  #3 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:266:4 04:19:48  #4 0x559bdfb32705 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::basic_string(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:451:9 04:19:48  #5 0x559bdfc326e6 in decltype(::new((void*)(0)) std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>(std::declval<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>())) std::construct_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:97:39 04:19:48  #6 0x559bdfc326e6 in void std::allocator_traits<std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::construct<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:514:4 04:19:48  #7 0x559bdfc326e6 in void std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::_M_realloc_insert<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(__gnu_cxx::__normal_iterator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/vector.tcc:449:4 04:19:48  #8 0x559be0032206 in doris::DeleteHandler::parse_condition(doris::DeleteSubPredicatePB const&, doris::TCondition*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:300:33 04:19:48  #9 0x559be0047aa3 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:355:9 04:19:48  #10 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13 04:19:48  #11 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28 04:19:48  #12 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18 04:19:48  #13 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18 04:19:48  #14 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5 04:19:48  #15 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32 04:19:48  #16 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5 04:19:48  #17 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21 04:19:48  #18 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31 04:19:48  #19 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14 04:19:48  #20 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2 04:19:48  #21 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9 04:19:48  #22 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24 04:19:48  #23 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5 04:19:48  #24 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8 ```
1 parent b92b63e commit b58570a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+623
-791
lines changed

‎be/src/cloud/cloud_tablet_mgr.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ class CloudTabletMgr::TabletMap {
136136
CloudTabletMgr::CloudTabletMgr(CloudStorageEngine& engine)
137137
: _engine(engine),
138138
_tablet_map(std::make_unique<TabletMap>()),
139-
_cache(std::make_unique<LRUCachePolicyTrackingManual>(
139+
_cache(std::make_unique<LRUCachePolicy>(
140140
CachePolicy::CacheType::CLOUD_TABLET_CACHE, config::tablet_cache_capacity,
141141
LRUCacheType::NUMBER, 0, config::tablet_cache_shards)) {}
142142

‎be/src/cloud/cloud_txn_delete_bitmap_cache.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@
3333
namespace doris {
3434

3535
CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
36-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE,
37-
size_in_bytes, LRUCacheType::SIZE, 86400, 4),
36+
: LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
37+
LRUCacheType::SIZE, 86400, 4),
3838
_stop_latch(1) {}
3939

4040
CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {

‎be/src/cloud/cloud_txn_delete_bitmap_cache.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
namespace doris {
3131

3232
// Record transaction related delete bitmaps using a lru cache.
33-
class CloudTxnDeleteBitmapCache : public LRUCachePolicyTrackingManual {
33+
class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
3434
public:
3535
CloudTxnDeleteBitmapCache(size_t size_in_bytes);
3636

‎be/src/http/default_path_handlers.cpp

+4-5
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ void display_tablets_callback(const WebPageHandler::ArgumentMap& args, EasyJson*
142142
// Registered to handle "/mem_tracker", and prints out memory tracker information.
143143
void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
144144
(*output) << "<h1>Memory usage by subsystem</h1>\n";
145-
std::vector<MemTracker::Snapshot> snapshots;
145+
std::vector<MemTrackerLimiter::Snapshot> snapshots;
146146
auto iter = args.find("type");
147147
if (iter != args.end()) {
148148
if (iter->second == "global") {
@@ -159,7 +159,7 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
159159
} else if (iter->second == "other") {
160160
MemTrackerLimiter::make_type_snapshots(&snapshots, MemTrackerLimiter::Type::OTHER);
161161
} else if (iter->second == "reserved_memory") {
162-
GlobalMemoryArbitrator::make_reserved_memory_snapshots(&snapshots);
162+
MemTrackerLimiter::make_all_reserved_trackers_snapshots(&snapshots);
163163
} else if (iter->second == "all") {
164164
MemTrackerLimiter::make_all_memory_state_snapshots(&snapshots);
165165
}
@@ -191,7 +191,6 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
191191
(*output) << "<thead><tr>"
192192
"<th data-sortable='true'>Type</th>"
193193
"<th data-sortable='true'>Label</th>"
194-
"<th data-sortable='true'>Parent Label</th>"
195194
"<th>Limit</th>"
196195
"<th data-sortable='true' "
197196
">Current Consumption(Bytes)</th>"
@@ -207,8 +206,8 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
207206
string peak_consumption_normalize = AccurateItoaKMGT(item.peak_consumption);
208207
(*output) << strings::Substitute(
209208
"<tr><td>$0</td><td>$1</td><td>$2</td><td>$3</td><td>$4</td><td>$5</td><td>$6</"
210-
"td><td>$7</td></tr>\n",
211-
item.type, item.label, item.parent_label, limit_str, item.cur_consumption,
209+
"td></tr>\n",
210+
item.type, item.label, limit_str, item.cur_consumption,
212211
current_consumption_normalize, item.peak_consumption, peak_consumption_normalize);
213212
}
214213
(*output) << "</tbody></table>\n";

‎be/src/olap/memtable.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,8 @@ MemTable::~MemTable() {
161161
std::for_each(_row_in_blocks.begin(), _row_in_blocks.end(), std::default_delete<RowInBlock>());
162162
_insert_mem_tracker->release(_mem_usage);
163163
_flush_mem_tracker->set_consumption(0);
164-
DCHECK_EQ(_insert_mem_tracker->consumption(), 0)
165-
<< std::endl
166-
<< MemTracker::log_usage(_insert_mem_tracker->make_snapshot());
164+
DCHECK_EQ(_insert_mem_tracker->consumption(), 0) << std::endl
165+
<< _insert_mem_tracker->log_usage();
167166
DCHECK_EQ(_flush_mem_tracker->consumption(), 0);
168167
_arena.reset();
169168
_agg_buffer_pool.clear();

‎be/src/olap/memtable_memory_limiter.cpp

+1-4
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,7 @@ Status MemTableMemoryLimiter::init(int64_t process_mem_limit) {
6262
_load_hard_mem_limit * config::load_process_safe_mem_permit_percent / 100;
6363
g_load_hard_mem_limit.set_value(_load_hard_mem_limit);
6464
g_load_soft_mem_limit.set_value(_load_soft_mem_limit);
65-
_memtable_tracker_set =
66-
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::LOAD, "MemTableTrackerSet");
67-
_mem_tracker = std::make_unique<MemTracker>("AllMemTableMemory",
68-
ExecEnv::GetInstance()->details_mem_tracker_set());
65+
_mem_tracker = std::make_unique<MemTracker>("AllMemTableMemory");
6966
REGISTER_HOOK_METRIC(memtable_memory_limiter_mem_consumption,
7067
[this]() { return _mem_tracker->consumption(); });
7168
_log_timer.start();

‎be/src/olap/memtable_memory_limiter.h

+1-4
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include <stdint.h>
2121

2222
#include "common/status.h"
23-
#include "runtime/memory/mem_tracker_limiter.h"
23+
#include "runtime/memory/mem_tracker.h"
2424
#include "util/countdown_latch.h"
2525
#include "util/stopwatch.hpp"
2626

@@ -45,7 +45,6 @@ class MemTableMemoryLimiter {
4545

4646
void refresh_mem_tracker();
4747

48-
MemTrackerLimiter* memtable_tracker_set() { return _memtable_tracker_set.get(); }
4948
MemTracker* mem_tracker() { return _mem_tracker.get(); }
5049

5150
int64_t mem_usage() const { return _mem_usage; }
@@ -68,8 +67,6 @@ class MemTableMemoryLimiter {
6867
int64_t _write_mem_usage = 0;
6968
int64_t _active_mem_usage = 0;
7069

71-
// mem tracker collection of all mem tables.
72-
std::shared_ptr<MemTrackerLimiter> _memtable_tracker_set;
7370
// sum of all mem table memory.
7471
std::unique_ptr<MemTracker> _mem_tracker;
7572
int64_t _load_hard_mem_limit = -1;

‎be/src/olap/memtable_writer.cpp

-13
Original file line numberDiff line numberDiff line change
@@ -187,25 +187,12 @@ Status MemTableWriter::wait_flush() {
187187
}
188188

189189
void MemTableWriter::_reset_mem_table() {
190-
#ifndef BE_TEST
191-
auto mem_table_insert_tracker = std::make_shared<MemTracker>(
192-
fmt::format("MemTableManualInsert:TabletId={}:MemTableNum={}#loadID={}",
193-
std::to_string(tablet_id()), _mem_table_num,
194-
UniqueId(_req.load_id).to_string()),
195-
ExecEnv::GetInstance()->memtable_memory_limiter()->memtable_tracker_set());
196-
auto mem_table_flush_tracker = std::make_shared<MemTracker>(
197-
fmt::format("MemTableHookFlush:TabletId={}:MemTableNum={}#loadID={}",
198-
std::to_string(tablet_id()), _mem_table_num++,
199-
UniqueId(_req.load_id).to_string()),
200-
ExecEnv::GetInstance()->memtable_memory_limiter()->memtable_tracker_set());
201-
#else
202190
auto mem_table_insert_tracker = std::make_shared<MemTracker>(fmt::format(
203191
"MemTableManualInsert:TabletId={}:MemTableNum={}#loadID={}",
204192
std::to_string(tablet_id()), _mem_table_num, UniqueId(_req.load_id).to_string()));
205193
auto mem_table_flush_tracker = std::make_shared<MemTracker>(fmt::format(
206194
"MemTableHookFlush:TabletId={}:MemTableNum={}#loadID={}", std::to_string(tablet_id()),
207195
_mem_table_num++, UniqueId(_req.load_id).to_string()));
208-
#endif
209196
{
210197
std::lock_guard<SpinLock> l(_mem_table_tracker_lock);
211198
_mem_table_insert_trackers.push_back(mem_table_insert_tracker);

‎be/src/olap/page_cache.h

+13-13
Original file line numberDiff line numberDiff line change
@@ -92,28 +92,28 @@ class StoragePageCache {
9292
}
9393
};
9494

95-
class DataPageCache : public LRUCachePolicyTrackingAllocator {
95+
class DataPageCache : public LRUCachePolicy {
9696
public:
9797
DataPageCache(size_t capacity, uint32_t num_shards)
98-
: LRUCachePolicyTrackingAllocator(
99-
CachePolicy::CacheType::DATA_PAGE_CACHE, capacity, LRUCacheType::SIZE,
100-
config::data_page_cache_stale_sweep_time_sec, num_shards) {}
98+
: LRUCachePolicy(CachePolicy::CacheType::DATA_PAGE_CACHE, capacity,
99+
LRUCacheType::SIZE, config::data_page_cache_stale_sweep_time_sec,
100+
num_shards) {}
101101
};
102102

103-
class IndexPageCache : public LRUCachePolicyTrackingAllocator {
103+
class IndexPageCache : public LRUCachePolicy {
104104
public:
105105
IndexPageCache(size_t capacity, uint32_t num_shards)
106-
: LRUCachePolicyTrackingAllocator(
107-
CachePolicy::CacheType::INDEXPAGE_CACHE, capacity, LRUCacheType::SIZE,
108-
config::index_page_cache_stale_sweep_time_sec, num_shards) {}
106+
: LRUCachePolicy(CachePolicy::CacheType::INDEXPAGE_CACHE, capacity,
107+
LRUCacheType::SIZE, config::index_page_cache_stale_sweep_time_sec,
108+
num_shards) {}
109109
};
110110

111-
class PKIndexPageCache : public LRUCachePolicyTrackingAllocator {
111+
class PKIndexPageCache : public LRUCachePolicy {
112112
public:
113113
PKIndexPageCache(size_t capacity, uint32_t num_shards)
114-
: LRUCachePolicyTrackingAllocator(
115-
CachePolicy::CacheType::PK_INDEX_PAGE_CACHE, capacity, LRUCacheType::SIZE,
116-
config::pk_index_page_cache_stale_sweep_time_sec, num_shards) {}
114+
: LRUCachePolicy(CachePolicy::CacheType::PK_INDEX_PAGE_CACHE, capacity,
115+
LRUCacheType::SIZE,
116+
config::pk_index_page_cache_stale_sweep_time_sec, num_shards) {}
117117
};
118118

119119
static constexpr uint32_t kDefaultNumShards = 16;
@@ -164,7 +164,7 @@ class StoragePageCache {
164164
// delete bitmap in unique key with mow
165165
std::unique_ptr<PKIndexPageCache> _pk_index_page_cache;
166166

167-
LRUCachePolicyTrackingAllocator* _get_page_cache(segment_v2::PageTypePB page_type) {
167+
LRUCachePolicy* _get_page_cache(segment_v2::PageTypePB page_type) {
168168
switch (page_type) {
169169
case segment_v2::DATA_PAGE: {
170170
return _data_page_cache.get();

‎be/src/olap/rowset/segment_v2/inverted_index_cache.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,9 @@ void InvertedIndexQueryCache::insert(const CacheKey& key, std::shared_ptr<roarin
135135
return;
136136
}
137137

138-
auto* lru_handle = LRUCachePolicyTrackingManual::insert(
139-
key.encode(), (void*)cache_value_ptr.release(), bitmap->getSizeInBytes(),
140-
bitmap->getSizeInBytes(), CachePriority::NORMAL);
138+
auto* lru_handle = LRUCachePolicy::insert(key.encode(), (void*)cache_value_ptr.release(),
139+
bitmap->getSizeInBytes(), bitmap->getSizeInBytes(),
140+
CachePriority::NORMAL);
141141
*handle = InvertedIndexQueryCacheHandle(this, lru_handle);
142142
}
143143

‎be/src/olap/rowset/segment_v2/inverted_index_cache.h

+15-16
Original file line numberDiff line numberDiff line change
@@ -99,23 +99,23 @@ class InvertedIndexSearcherCache {
9999
private:
100100
InvertedIndexSearcherCache() = default;
101101

102-
class InvertedIndexSearcherCachePolicy : public LRUCachePolicyTrackingManual {
102+
class InvertedIndexSearcherCachePolicy : public LRUCachePolicy {
103103
public:
104104
InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
105105
uint32_t element_count_capacity)
106-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE,
107-
capacity, LRUCacheType::SIZE,
108-
config::inverted_index_cache_stale_sweep_time_sec,
109-
num_shards, element_count_capacity, true) {}
106+
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
107+
LRUCacheType::SIZE,
108+
config::inverted_index_cache_stale_sweep_time_sec, num_shards,
109+
element_count_capacity, true) {}
110110
InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
111111
uint32_t element_count_capacity,
112112
CacheValueTimeExtractor cache_value_time_extractor,
113113
bool cache_value_check_timestamp)
114-
: LRUCachePolicyTrackingManual(
115-
CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
116-
LRUCacheType::SIZE, config::inverted_index_cache_stale_sweep_time_sec,
117-
num_shards, element_count_capacity, cache_value_time_extractor,
118-
cache_value_check_timestamp, true) {}
114+
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
115+
LRUCacheType::SIZE,
116+
config::inverted_index_cache_stale_sweep_time_sec, num_shards,
117+
element_count_capacity, cache_value_time_extractor,
118+
cache_value_check_timestamp, true) {}
119119
};
120120
// Insert a cache entry by key.
121121
// And the cache entry will be returned in handle.
@@ -179,9 +179,9 @@ class InvertedIndexCacheHandle {
179179

180180
class InvertedIndexQueryCacheHandle;
181181

182-
class InvertedIndexQueryCache : public LRUCachePolicyTrackingManual {
182+
class InvertedIndexQueryCache : public LRUCachePolicy {
183183
public:
184-
using LRUCachePolicyTrackingManual::insert;
184+
using LRUCachePolicy::insert;
185185

186186
// cache key
187187
struct CacheKey {
@@ -227,10 +227,9 @@ class InvertedIndexQueryCache : public LRUCachePolicyTrackingManual {
227227
InvertedIndexQueryCache() = delete;
228228

229229
InvertedIndexQueryCache(size_t capacity, uint32_t num_shards)
230-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE,
231-
capacity, LRUCacheType::SIZE,
232-
config::inverted_index_cache_stale_sweep_time_sec,
233-
num_shards) {}
230+
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE, capacity,
231+
LRUCacheType::SIZE, config::inverted_index_cache_stale_sweep_time_sec,
232+
num_shards) {}
234233

235234
bool lookup(const CacheKey& key, InvertedIndexQueryCacheHandle* handle);
236235

‎be/src/olap/schema_cache.h

+3-4
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ using SegmentIteratorUPtr = std::unique_ptr<SegmentIterator>;
4444
// eliminating the need for frequent allocation and deallocation during usage.
4545
// This caching mechanism proves immensely advantageous, particularly in scenarios
4646
// with high concurrency, where queries are executed simultaneously.
47-
class SchemaCache : public LRUCachePolicyTrackingManual {
47+
class SchemaCache : public LRUCachePolicy {
4848
public:
4949
static SchemaCache* instance();
5050

@@ -86,9 +86,8 @@ class SchemaCache : public LRUCachePolicyTrackingManual {
8686
};
8787

8888
SchemaCache(size_t capacity)
89-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::SCHEMA_CACHE, capacity,
90-
LRUCacheType::NUMBER,
91-
config::schema_cache_sweep_time_sec) {}
89+
: LRUCachePolicy(CachePolicy::CacheType::SCHEMA_CACHE, capacity, LRUCacheType::NUMBER,
90+
config::schema_cache_sweep_time_sec) {}
9291

9392
private:
9493
static constexpr char SCHEMA_DELIMITER = '-';

‎be/src/olap/segment_loader.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ bool SegmentCache::lookup(const SegmentCache::CacheKey& key, SegmentCacheHandle*
4040

4141
void SegmentCache::insert(const SegmentCache::CacheKey& key, SegmentCache::CacheValue& value,
4242
SegmentCacheHandle* handle) {
43-
auto* lru_handle = LRUCachePolicyTrackingManual::insert(
44-
key.encode(), &value, value.segment->meta_mem_usage(), value.segment->meta_mem_usage(),
45-
CachePriority::NORMAL);
43+
auto* lru_handle =
44+
LRUCachePolicy::insert(key.encode(), &value, value.segment->meta_mem_usage(),
45+
value.segment->meta_mem_usage(), CachePriority::NORMAL);
4646
handle->push_segment(this, lru_handle);
4747
}
4848

‎be/src/olap/segment_loader.h

+5-6
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ class BetaRowset;
5555
// Make sure that cache_handle is valid during the segment usage period.
5656
using BetaRowsetSharedPtr = std::shared_ptr<BetaRowset>;
5757

58-
class SegmentCache : public LRUCachePolicyTrackingManual {
58+
class SegmentCache : public LRUCachePolicy {
5959
public:
60-
using LRUCachePolicyTrackingManual::insert;
60+
using LRUCachePolicy::insert;
6161
// The cache key or segment lru cache
6262
struct CacheKey {
6363
CacheKey(RowsetId rowset_id_, int64_t segment_id_)
@@ -81,10 +81,9 @@ class SegmentCache : public LRUCachePolicyTrackingManual {
8181
};
8282

8383
SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)
84-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::SEGMENT_CACHE,
85-
memory_bytes_limit, LRUCacheType::SIZE,
86-
config::tablet_rowset_stale_sweep_time_sec,
87-
DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {}
84+
: LRUCachePolicy(CachePolicy::CacheType::SEGMENT_CACHE, memory_bytes_limit,
85+
LRUCacheType::SIZE, config::tablet_rowset_stale_sweep_time_sec,
86+
DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {}
8887

8988
// Lookup the given segment in the cache.
9089
// If the segment is found, the cache entry will be written into handle.

‎be/src/olap/storage_engine.h

+4-4
Original file line numberDiff line numberDiff line change
@@ -540,7 +540,7 @@ class StorageEngine final : public BaseStorageEngine {
540540
// lru cache for create tabelt round robin in disks
541541
// key: partitionId_medium
542542
// value: index
543-
class CreateTabletIdxCache : public LRUCachePolicyTrackingManual {
543+
class CreateTabletIdxCache : public LRUCachePolicy {
544544
public:
545545
// get key, delimiter with DELIMITER '-'
546546
static std::string get_key(int64_t partition_id, TStorageMedium::type medium) {
@@ -558,9 +558,9 @@ class CreateTabletIdxCache : public LRUCachePolicyTrackingManual {
558558
};
559559

560560
CreateTabletIdxCache(size_t capacity)
561-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::CREATE_TABLET_RR_IDX_CACHE,
562-
capacity, LRUCacheType::NUMBER,
563-
/*stale_sweep_time_s*/ 30 * 60) {}
561+
: LRUCachePolicy(CachePolicy::CacheType::CREATE_TABLET_RR_IDX_CACHE, capacity,
562+
LRUCacheType::NUMBER,
563+
/*stale_sweep_time_s*/ 30 * 60) {}
564564
};
565565

566566
struct DirInfo {

‎be/src/olap/tablet_manager.cpp

+1-2
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,7 @@ bvar::Adder<int64_t> g_tablet_meta_schema_columns_count("tablet_meta_schema_colu
9090

9191
TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size)
9292
: _engine(engine),
93-
_tablet_meta_mem_tracker(std::make_shared<MemTracker>(
94-
"TabletMeta(experimental)", ExecEnv::GetInstance()->details_mem_tracker_set())),
93+
_tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta(experimental)")),
9594
_tablets_shards_size(tablet_map_lock_shard_size),
9695
_tablets_shards_mask(tablet_map_lock_shard_size - 1) {
9796
CHECK_GT(_tablets_shards_size, 0);

‎be/src/olap/tablet_meta.h

+4-5
Original file line numberDiff line numberDiff line change
@@ -520,13 +520,12 @@ class DeleteBitmap {
520520

521521
void remove_sentinel_marks();
522522

523-
class AggCachePolicy : public LRUCachePolicyTrackingManual {
523+
class AggCachePolicy : public LRUCachePolicy {
524524
public:
525525
AggCachePolicy(size_t capacity)
526-
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE,
527-
capacity, LRUCacheType::SIZE,
528-
config::delete_bitmap_agg_cache_stale_sweep_time_sec,
529-
256) {}
526+
: LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity,
527+
LRUCacheType::SIZE,
528+
config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {}
530529
};
531530

532531
class AggCache {

0 commit comments

Comments
 (0)