[active-standby] support reset heartbeat suspend timer

src/DbInterface.cpp

@@ -28,6 +28,7 @@
 #include <boost/bind/bind.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/tokenizer.hpp>
 
 #include "swss/netdispatcher.h"
 #include "swss/netlink.h"
@@ -1050,6 +1051,10 @@ void DbInterface::processMuxLinkmgrConfigNotifiction(std::deque<swss::KeyOpField
                         mMuxManagerPtr->processSrcMac(v == "ToRMac");
                     } else if (f == "interval_pck_loss_count_update") {
                         mMuxManagerPtr->setLinkProberStatUpdateIntervalCount(boost::lexical_cast<uint32_t> (v));
+                    } else if (f == "reset_suspend_timer") {
+                        boost::tokenizer<> tok(v);
+                        std::vector<std::string> ports(tok.begin(), tok.end());
+                        mMuxManagerPtr->processResetSuspendTimer(ports);
                     }
 
                     MUXLOGINFO(boost::format("key: %s, Operation: %s, f: %s, v: %s") %

src/MuxManager.cpp

@@ -598,4 +598,21 @@ void MuxManager::handleTsaEnableNotification(bool enable)
     }
 }
 
+//
+// ---> processResetSuspendTimer
+//
+// process suspend timer reset requests
+//
+void MuxManager::processResetSuspendTimer(const std::vector<std::string> &portNames)
+{
+    for (const std::string &portName : portNames)
+    {
+        MUXLOGINFO(boost::format("%s: reset heartbeat suspend timer") % portName);
+        PortMapIterator portMapIterator = mPortMap.find(portName);
+        if (portMapIterator != mPortMap.end()) {
+            portMapIterator->second->handleResetSuspendTimer();
+        }
+    }
+}
+
 } /* namespace mux */

src/MuxManager.h

@@ -495,6 +495,17 @@ class MuxManager
      */
     void handleTsaEnableNotification(bool enable);
 
+    /**
+    *@method processResetSuspendTimer
+    *
+    *@brief process suspend timer reset requests
+    *
+    *@param portNames (in)  mux ports to reset the suspend timer
+    *
+    *@return none
+    */
+    void processResetSuspendTimer(const std::vector<std::string> &portNames);
+
 private:
     /**
     *@method getMuxPortCableType

src/MuxPort.cpp

@@ -437,4 +437,19 @@ void MuxPort::handleTsaEnable(bool enable)
     ));
 }
 
+//
+// ---> handleResetSuspendTimer();
+//
+// handle suspend timer reset
+//
+void MuxPort::handleResetSuspendTimer()
+{
+    MUXLOGWARNING(boost::format("%s: reset heartbeat suspend timer") % mMuxPortConfig.getPortName());
+
+    boost::asio::post(mStrand, boost::bind(
+        &link_manager::LinkManagerStateMachineBase::handleResetSuspendTimer,
+        mLinkManagerStateMachinePtr.get()
+    ));
+}
+
 } /* namespace mux */

src/MuxPort.h

@@ -419,6 +419,15 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
      */
     void handleTsaEnable(bool enable);
 
+    /**
+     *@method handleResetSuspendTimer
+     *
+     *@brief handle reset suspend timer request
+     *
+     * @return none
+     */
+    void handleResetSuspendTimer();
+
 protected:
     friend class test::MuxManagerTest;
     friend class test::FakeMuxPort;

src/link_manager/LinkManagerStateMachineActiveStandby.cpp

@@ -958,6 +958,22 @@ void ActiveStandbyStateMachine::handleResetLinkProberPckLossCount()
     mResetIcmpPacketCountsFnPtr();
 }
 
+// ---> handleResetSuspendTimer();
+//
+// reset the heartbeat suspend timer
+//
+void ActiveStandbyStateMachine::handleResetSuspendTimer()
+{
+    MUXLOGDEBUG(boost::format("%s: reset heartbeat suspend timer") % mMuxPortConfig.getPortName());
+
+    if (ps(mCompositeState) == link_prober::LinkProberState::Label::Unknown &&
+        ms(mCompositeState) == mux_state::MuxState::Label::Active &&
+        ls(mCompositeState) == link_state::LinkState::Label::Up) {
+        mUnknownActiveUpBackoffFactor = 1;
+        mResumeTxFnPtr();
+    }
+}
+
 //
 // ---> updateMuxLinkmgrState();
 //

src/link_manager/LinkManagerStateMachineActiveStandby.h

@@ -401,6 +401,15 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
     */
     void handleResetLinkProberPckLossCount();
 
+    /**
+     * @method handleResetSuspendTimer
+     *
+     * @brief reset the heartbeat suspend timer
+     *
+     * @return none
+     */
+    void handleResetSuspendTimer();
+
 private:
     /**
     *@method updateMuxLinkmgrState

src/link_manager/LinkManagerStateMachineBase.cpp

@@ -336,6 +336,15 @@ void LinkManagerStateMachineBase::handleResetLinkProberPckLossCount()
     MUXLOGINFO(mMuxPortConfig.getPortName());
 }
 
+// ---> handleResetSuspendTimer();
+//
+// reset the heartbeat suspend timer
+//
+void LinkManagerStateMachineBase::handleResetSuspendTimer()
+{
+    MUXLOGINFO(mMuxPortConfig.getPortName());
+}
+
 //
 // ---> postMuxStateEvent(mux_state::MuxState::Label label)
 //

src/link_manager/LinkManagerStateMachineBase.h

@@ -505,6 +505,15 @@ class LinkManagerStateMachineBase : public common::StateMachine {
      */
     virtual void handleResetLinkProberPckLossCount();
 
+    /**
+     * @method handleResetSuspendTimer
+     *
+     * @brief reset the heartbeat suspend timer
+     *
+     * @return none
+     */
+    virtual void handleResetSuspendTimer();
+
 public:
     /**
     *@method getLinkProberStateMachinePtr

src/link_prober/LinkProber.cpp

@@ -188,8 +188,9 @@ void LinkProber::startProbing()
 //
 void LinkProber::suspendTxProbes(uint32_t suspendTime_msec)
 {
-    MUXLOGWARNING(boost::format("%s: suspend ICMP heartbeat probing") % mMuxPortConfig.getPortName());
+    MUXLOGWARNING(boost::format("%s: suspend ICMP heartbeat probing %dms") % mMuxPortConfig.getPortName() % suspendTime_msec);
 
+    // NOTE: the timer reset also cancels any pending async ops with ec as boost::asio::error::operation_aborted
     mSuspendTimer.expires_from_now(boost::posix_time::milliseconds(suspendTime_msec));
     mSuspendTimer.async_wait(mStrand.wrap(boost::bind(
         &LinkProber::handleSuspendTimeout,
@@ -198,6 +199,7 @@ void LinkProber::suspendTxProbes(uint32_t suspendTime_msec)
     )));
 
     mSuspendTx = true;
+    mCancelSuspend = false;
 }
 
 //
@@ -210,6 +212,7 @@ void LinkProber::resumeTxProbes()
     MUXLOGWARNING(boost::format("%s: resume ICMP heartbeat probing") % mMuxPortConfig.getPortName());
 
     mSuspendTimer.cancel();
+    mCancelSuspend = true;
 }
 
 //
@@ -538,8 +541,8 @@ void LinkProber::handleSuspendTimeout(boost::system::error_code errorCode)
 
     mSuspendTx = false;
 
-    if (errorCode == boost::system::errc::success) {
-        // inform the composite state machine about Suspend timer expiry
+    if (errorCode == boost::system::errc::success || mCancelSuspend) {
+        // inform the composite state machine about Suspend timer expiry or cancel
         boost::asio::io_service::strand &strand = mLinkProberStateMachinePtr->getStrand();
         boost::asio::io_service &ioService = strand.context();
         ioService.post(strand.wrap(boost::bind(
@@ -549,6 +552,8 @@ void LinkProber::handleSuspendTimeout(boost::system::error_code errorCode)
             LinkProberStateMachineBase::getSuspendTimerExpiredEvent()
         )));
     }
+
+    mCancelSuspend = false;
 }
 
 //

src/link_prober/LinkProber.h

@@ -629,6 +629,7 @@ class LinkProber
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> mTxBuffer;
     std::array<uint8_t, MUX_MAX_ICMP_BUFFER_SIZE> mRxBuffer;
 
+    bool mCancelSuspend = false;
     bool mSuspendTx = false;
     bool mShutdownTx = false;
     bool mDecreaseProbingInterval = false;

test/FakeMuxPort.h

@@ -73,6 +73,7 @@ class FakeMuxPort : public ::mux::MuxPort {
 
     link_prober::LinkProberState::Label getPeerLinkProberState() { return getActiveActiveStateMachinePtr()->mPeerLinkProberState; };
     mux_state::MuxState::Label getPeerMuxState() { return getActiveActiveStateMachinePtr()->mPeerMuxState; };
+    uint32_t getActiveStandbyStateMachineSuspendBackoffFactor() { return getActiveStandbyStateMachinePtr()->mUnknownActiveUpBackoffFactor; }
 
     inline void initLinkProberActiveActive();
     inline void initLinkProberActiveStandby();

test/LinkManagerStateMachineActiveActiveTest.cpp

@@ -223,6 +223,12 @@ void LinkManagerStateMachineActiveActiveTest::handleMuxConfig(std::string config
     }
 }
 
+void LinkManagerStateMachineActiveActiveTest::handleResetSuspendTimer(uint32_t count)
+{
+    mFakeMuxPort.handleResetSuspendTimer();
+    runIoService(count);
+}
+
 void LinkManagerStateMachineActiveActiveTest::activateStateMachine(bool enable_feature_default_route)
 {
     mMuxConfig.enableDefaultRouteFeature(enable_feature_default_route);
@@ -1210,4 +1216,17 @@ TEST_F(LinkManagerStateMachineActiveActiveTest, StateMachineInitEventWithMultipl
     EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 2);
 }
 
+TEST_F(LinkManagerStateMachineActiveActiveTest, ResetSuspendTimer)
+{
+    setMuxActive();
+
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 2);
+    // reset suspend timer is NOOP for active-active composite state machine
+    handleResetSuspendTimer();
+    VALIDATE_STATE(Active, Active, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 2);
+}
+
 } /* namespace test */

test/LinkManagerStateMachineActiveActiveTest.h

@@ -43,6 +43,7 @@ class LinkManagerStateMachineActiveActiveTest : public ::testing::Test
     void handleProbeMuxState(std::string, uint32_t count = 0);
     void handleLinkState(std::string linkState, uint32_t count = 0);
     void handleMuxConfig(std::string config, uint32_t count = 0, bool poll = false);
+    void handleResetSuspendTimer(uint32_t count = 0);
     void activateStateMachine(bool enable_feature_default_route=false);
     void setMuxActive();
     void setMuxStandby();

test/LinkManagerStateMachineTest.cpp

@@ -211,6 +211,12 @@ void LinkManagerStateMachineTest::handleMuxConfig(std::string config, uint32_t c
     runIoService(count);
 }
 
+void LinkManagerStateMachineTest::handleResetSuspendTimer(uint32_t count)
+{
+    mFakeMuxPort.handleResetSuspendTimer();
+    runIoService(count);
+}
+
 void LinkManagerStateMachineTest::activateStateMachine()
 {
     mFakeMuxPort.activateStateMachine();
@@ -1607,4 +1613,43 @@ TEST_F(LinkManagerStateMachineTest, DefaultRouteStateRaceCondition)
     stopIoServiceThreaded();
 }
 
+TEST_F(LinkManagerStateMachineTest, ResetSuspendTimer)
+{
+    setMuxActive();
+
+    // swss notification
+    handleMuxState("active", 3);
+    VALIDATE_STATE(Active, Active, Up);
+
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+
+    handleResetSuspendTimer();
+
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 0);
+    EXPECT_EQ(mDbInterfacePtr->mGetMuxStateInvokeCount, 1);
+    postLinkProberEvent(link_prober::LinkProberState::Unknown, 2);
+    VALIDATE_STATE(Unknown, Active, Up);
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.getActiveStandbyStateMachineSuspendBackoffFactor(), 2);
+
+    handleResetSuspendTimer();
+
+    // once suspend timer is reset, the tx probe is resumed
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 2);
+    // the backoff factor is restored to 1
+    EXPECT_EQ(mFakeMuxPort.getActiveStandbyStateMachineSuspendBackoffFactor(), 1);
+
+    postSuspendTimerExpiredEvent(2);
+    VALIDATE_STATE(Unknown, Wait, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 1);
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 1);
+}
+
 } /* namespace test */

test/LinkManagerStateMachineTest.h

@@ -51,6 +51,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void handleProbeMuxState(std::string, uint32_t count = 0);
     void handleLinkState(std::string linkState, uint32_t count = 0);
     void handleMuxConfig(std::string config, uint32_t count = 0);
+    void handleResetSuspendTimer(uint32_t count = 0);
     void activateStateMachine();
     void setMuxActive();
     void setMuxStandby();

test/MuxManagerTest.cpp

@@ -903,9 +903,11 @@ TEST_F(MuxManagerTest, LinkmgrdConfig)
         {"LINK_PROBER", "SET", {{"interval_v4", "abc"}}},
         {"LINK_PROBER", "SET", {{"src_mac", "ToRMac"}}},
         {"LINK_PROBER", "SET", {{"interval_pck_loss_count_update", "900"}}},
+        {"LINK_PROBER", "SET", {{"reset_suspend_timer", "Ethernet0"}}},
         {"MUXLOGGER", "SET", {{"log_verbosity", "warning"}}},
     };
     processMuxLinkmgrConfigNotifiction(entries);
+    runIoService(2);
 
     EXPECT_TRUE(getTimeoutIpv4_msec(port) == v4PorbeInterval);
     EXPECT_TRUE(getTimeoutIpv6_msec(port) == v6ProveInterval);