From 7596282a556accb84fbcec3f31a3e2017d8ade0c Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:03:47 +0100 Subject: [PATCH 1/5] memusage: Allow counting usage of vectors with different allocators --- src/memusage.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memusage.h b/src/memusage.h index 08be66172e..43407507ed 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -84,10 +84,10 @@ struct stl_shared_counter size_t weak_count; }; -template -static inline size_t DynamicUsage(const std::vector& v) +template +static inline size_t DynamicUsage(const std::vector& v) { - return MallocUsage(v.capacity() * sizeof(X)); + return MallocUsage(v.capacity() * sizeof(T)); } template From c6594c0b142133535c1d2d5b8d8084cf9e57592b Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:19:00 +0100 Subject: [PATCH 2/5] memusage: Add DynamicUsage for std::string Add DynamicUsage(std::string) which Returns the dynamic allocation of a std::string, or 0 if none (in case of small string optimization). --- src/memusage.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/memusage.h b/src/memusage.h index 43407507ed..9d9e549ef2 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,18 @@ static inline size_t DynamicUsage(const std::vector& v) return MallocUsage(v.capacity() * sizeof(T)); } +static inline size_t DynamicUsage(const std::string& s) +{ + const char* s_ptr = reinterpret_cast(&s); + // Don't count the dynamic memory used for string, if it resides in the + // "small string" optimization area (which stores data inside the object itself, up to some + // size; 15 bytes in modern libstdc++). + if (!std::less{}(s.data(), s_ptr) && !std::greater{}(s.data() + s.size(), s_ptr + sizeof(s))) { + return 0; + } + return MallocUsage(s.capacity()); +} + template static inline size_t DynamicUsage(const prevector& v) { From c3a6722f34a575834b282894d441d7bf802bd467 Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:26:06 +0100 Subject: [PATCH 3/5] net: Use DynamicUsage(m_type) in CSerializedNetMsg::GetMemoryUsage Now that memusage correctly computes the dynamic size of a string, there is no need for special handling here. --- src/net.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index d76b39f75d..3e936b5f3d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -121,10 +121,7 @@ std::string strSubVersion; size_t CSerializedNetMsg::GetMemoryUsage() const noexcept { - // Don't count the dynamic memory used for the m_type string, by assuming it fits in the - // "small string" optimization area (which stores data inside the object itself, up to some - // size; 15 bytes in modern libstdc++). - return sizeof(*this) + memusage::DynamicUsage(data); + return sizeof(*this) + memusage::DynamicUsage(m_type) + memusage::DynamicUsage(data); } void CConnman::AddAddrFetch(const std::string& strDest) From 047b5e2af1f03549d85926aa02fed0dfa00d449f Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:04:54 +0100 Subject: [PATCH 4/5] streams: add DataStream::GetMemoryUsage --- src/streams.cpp | 6 ++++++ src/streams.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/streams.cpp b/src/streams.cpp index baa5ad7abe..cd496ee2be 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/license/mit/. +#include #include #include #include @@ -110,3 +111,8 @@ bool AutoFile::Truncate(unsigned size) { return ::TruncateFile(m_file, size); } + +size_t DataStream::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(vch); +} diff --git a/src/streams.h b/src/streams.h index e9f3562c6c..64dbfbff14 100644 --- a/src/streams.h +++ b/src/streams.h @@ -277,6 +277,9 @@ public: { util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); } + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; template From d22a234ed270286b483aec2db1e2f716b9756231 Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:05:20 +0100 Subject: [PATCH 5/5] net: Use actual memory size in receive buffer accounting Add a method CNetMessage::GetMemoryUsage and use this for accounting of the size of the process receive queue instead of the raw message size. This ensures that allocation and deserialization overhead is taken into account. --- src/net.cpp | 9 +++++++-- src/net.h | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 3e936b5f3d..f1ac681b99 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -124,6 +124,11 @@ size_t CSerializedNetMsg::GetMemoryUsage() const noexcept return sizeof(*this) + memusage::DynamicUsage(m_type) + memusage::DynamicUsage(data); } +size_t CNetMessage::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(m_type) + m_recv.GetMemoryUsage(); +} + void CConnman::AddAddrFetch(const std::string& strDest) { LOCK(m_addr_fetches_mutex); @@ -3769,7 +3774,7 @@ void CNode::MarkReceivedMsgsForProcessing() for (const auto& msg : vRecvMsg) { // vRecvMsg contains only completed CNetMessage // the single possible partially deserialized message are held by TransportDeserializer - nSizeAdded += msg.m_raw_message_size; + nSizeAdded += msg.GetMemoryUsage(); } LOCK(m_msg_process_queue_mutex); @@ -3786,7 +3791,7 @@ std::optional> CNode::PollMessage() std::list msgs; // Just take one message msgs.splice(msgs.begin(), m_msg_process_queue, m_msg_process_queue.begin()); - m_msg_process_queue_size -= msgs.front().m_raw_message_size; + m_msg_process_queue_size -= msgs.front().GetMemoryUsage(); fPauseRecv = m_msg_process_queue_size > m_recv_flood_size; return std::make_pair(std::move(msgs.front()), !m_msg_process_queue.empty()); diff --git a/src/net.h b/src/net.h index 6b63e2cc0d..20c654c34a 100644 --- a/src/net.h +++ b/src/net.h @@ -245,6 +245,9 @@ public: CNetMessage(const CNetMessage&) = delete; CNetMessage& operator=(CNetMessage&&) = default; CNetMessage& operator=(const CNetMessage&) = delete; + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; /** The Transport converts one connection's sent messages to wire bytes, and received bytes back. */