From ad96646e1d97cff67b81cc828d94f038a433c9f2 Mon Sep 17 00:00:00 2001 From: Kavitha Ramalingam Date: Thu, 14 Nov 2024 16:21:30 +0530 Subject: [PATCH] gNOI Warm Reboot - Added tests --- .../rebootbackend/rebootbe.cpp | 9 +- .../tests/reboot_thread_test.cpp | 65 +++++++ src/sonic-framework/tests/rebootbe_test.cpp | 179 +++++++++++++++++- 3 files changed, 246 insertions(+), 7 deletions(-) diff --git a/src/sonic-framework/rebootbackend/rebootbe.cpp b/src/sonic-framework/rebootbackend/rebootbe.cpp index e283eec90a19..9a40f01359d1 100644 --- a/src/sonic-framework/rebootbackend/rebootbe.cpp +++ b/src/sonic-framework/rebootbackend/rebootbe.cpp @@ -31,7 +31,6 @@ RebootBE::RebootBE(DbusInterface &dbus_interface) } RebootBE::RebManagerStatus RebootBE::GetCurrentStatus() { - const std::lock_guard lock(m_StatusMutex); return m_CurrentStatus; } @@ -43,6 +42,9 @@ void RebootBE::SetCurrentStatus(RebManagerStatus newStatus) { void RebootBE::Start() { SWSS_LOG_ENTER(); SWSS_LOG_NOTICE("--- Starting rebootbackend ---"); + swss::WarmStart::initialize("rebootbackend", "sonic-framework"); + swss::WarmStart::checkWarmStart("rebootbackend", "sonic-framework", + /*incr_restore_cnt=*/false); swss::Select s; s.addSelectable(&m_NotificationConsumer); @@ -51,7 +53,6 @@ void RebootBE::Start() { if (swss::WarmStart::isWarmStart()) { - SWSS_LOG_NOTICE("Launching init thread for warm start"); SetCurrentStatus(RebManagerStatus::WARM_INIT_WAIT); } else { SWSS_LOG_NOTICE("Warm restart not enabled"); @@ -176,9 +177,9 @@ bool RebootBE::RebootAllowed(const gnoi::system::RebootMethod rebMethod) { case RebManagerStatus::WARM_REBOOT_IN_PROGRESS: { return false; } - case RebManagerStatus::WARM_INIT_WAIT: { + case RebManagerStatus::WARM_INIT_WAIT: { return rebMethod == gnoi::system::RebootMethod::COLD; - } + } case RebManagerStatus::IDLE: { return true; } diff --git a/src/sonic-framework/tests/reboot_thread_test.cpp b/src/sonic-framework/tests/reboot_thread_test.cpp index bbdb2954ac6b..7fffc3342cb0 100644 --- a/src/sonic-framework/tests/reboot_thread_test.cpp +++ b/src/sonic-framework/tests/reboot_thread_test.cpp @@ -86,6 +86,34 @@ TEST_F(RebootStatusTest, TestGetStatus) { EXPECT_EQ(0, response.when()); } +TEST_F(RebootStatusTest, TestGetWarmStatus) { + std::chrono::nanoseconds curr_ns = std::chrono::high_resolution_clock::now().time_since_epoch(); + + m_status.set_start_status(gnoi::system::RebootMethod::WARM, "reboot because"); + + gnoi::system::RebootStatusResponse response = m_status.get_response(); + EXPECT_EQ(response.status().status(), + gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN); + + m_status.set_completed_status( + gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS, "anything"); + + response = m_status.get_response(); + + // message should be empty while reboot is active + EXPECT_THAT(response.status().message(), StrEq("")); + + uint64_t reboot_ns = response.when(); + EXPECT_TRUE(reboot_ns > (uint64_t)curr_ns.count()); + + m_status.set_inactive(); + response = m_status.get_response(); + EXPECT_THAT(response.status().message(), StrEq("anything")); + EXPECT_EQ(response.status().status(), + gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_SUCCESS); + EXPECT_EQ(0, response.when()); +} + class RebootThreadTest : public ::testing::Test { protected: RebootThreadTest() @@ -278,4 +306,41 @@ TEST_F(RebootThreadTest, TestInvalidMethodfDoReboot) { IsStatus(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_F(RebootThreadTest, TestNoWarmIfNonRetriableFailure) { + set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot"); + set_completed_status(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "failed to warm reboot"); + force_inactive(); + + gnoi::system::RebootRequest request; + request.set_method(gnoi::system::RebootMethod::WARM); + + NotificationResponse response = m_reboot_thread.Start(request); + EXPECT_EQ(response.status, swss::StatusCode::SWSS_RC_FAILED_PRECONDITION); + EXPECT_EQ(response.json_string, + "RebootThread: last WARM reboot failed with non-retriable failure"); +} + +TEST_F(RebootThreadTest, TestSigTermStartofDoReboot) { + sigterm_requested = true; + set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot"); + do_reboot(); + force_inactive(); + gnoi::system::RebootStatusResponse response = m_reboot_thread.GetResponse(); + EXPECT_THAT( + response, + IsStatus(gnoi::system::RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + +TEST_F(RebootThreadTest, TestWaitForRebootPositive) { + overwrite_reboot_timeout(1); + set_start_status(gnoi::system::RebootMethod::WARM, "time to reboot"); + swss::Select s; + swss::SelectableEvent m_stop; + s.addSelectable(&m_stop); + RebootThread::Progress progress = wait_for_platform_reboot(s); + EXPECT_EQ(progress, RebootThread::Progress::PROCEED); +} + + } // namespace rebootbackend diff --git a/src/sonic-framework/tests/rebootbe_test.cpp b/src/sonic-framework/tests/rebootbe_test.cpp index bc575a1d7b99..d5a75f66ffd1 100644 --- a/src/sonic-framework/tests/rebootbe_test.cpp +++ b/src/sonic-framework/tests/rebootbe_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include "mock_reboot_interfaces.h" #include "reboot_common.h" @@ -63,6 +62,7 @@ class RebootBETestWithoutStop : public ::testing::Test { m_rebootbeReponseChannel(&m_db, REBOOT_RESPONSE_NOTIFICATION_CHANNEL), m_rebootbe(m_dbus_interface) { sigterm_requested = false; +// TestUtils::clear_tables(m_db); m_s.addSelectable(&m_rebootbeReponseChannel); @@ -74,6 +74,15 @@ class RebootBETestWithoutStop : public ::testing::Test { } virtual ~RebootBETestWithoutStop() = default; + void force_warm_start_state(bool enabled) { + swss::Table enable_table(&m_db, STATE_WARM_RESTART_ENABLE_TABLE_NAME); + enable_table.hset("system", "enable", enabled ? "true" : "false"); + enable_table.hset("sonic-framework", "enable", enabled ? "true" : "false"); + + swss::Table restart_table(&m_db, STATE_WARM_RESTART_TABLE_NAME); + restart_table.hset("rebootbackend", "restore_count", enabled ? "0" : ""); + } + void start_rebootbe() { m_rebootbe_thread = std::make_unique(&RebootBE::Start, &m_rebootbe); @@ -202,6 +211,50 @@ class RebootBETest : public RebootBETestWithoutStop { } }; +TEST_F(RebootBETest, WarmbootInProgressBlocksNewWarmboot) { + force_warm_start_state(true); + + start_rebootbe(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + + // Send a warmboot request, confirm it fails. + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request, swss::StatusCode::SWSS_RC_IN_USE); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + force_warm_start_state(false); +} + + +TEST_F(RebootBETest, ColdbootWhileWarmbootInProgress) { + force_warm_start_state(true); + set_mock_defaults(); + + start_rebootbe(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_INIT_WAIT); + + // Send a coldboot request, confirm it starts. + RebootRequest request; + request.set_method(RebootMethod::COLD); + start_reboot_via_rpc(request); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::COLD_REBOOT_IN_PROGRESS); + + // Cleanup without going through the whole reboot. + send_stop_reboot_thread(); + force_warm_start_state(false); +} + + // Test fixture to skip through the startup sequence into the main loop. // Param indicates if RebootBE should be initialized into a state where the // system came up in warmboot. @@ -209,11 +262,11 @@ class RebootBEAutoStartTest : public RebootBETest, public ::testing::WithParamInterface { protected: RebootBEAutoStartTest() { - //force_warm_start_state(GetParam()); start_rebootbe(); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + //std::this_thread::sleep_for(std::chrono::milliseconds(50)); + std::this_thread::sleep_for(std::chrono::milliseconds(ONE_SECOND_MS)); EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); } }; @@ -312,6 +365,31 @@ TEST_P(RebootBEAutoStartTest, TestColdRebootDbusToCompletion) { "platform failed to reboot")); } + +TEST_P(RebootBEAutoStartTest, TestWarmRebootDbusToCompletion) { + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_SUCCESS, ""}; + EXPECT_CALL(m_dbus_interface, Reboot(_)) + .Times(1) + .WillRepeatedly(Return(dbus_response)); + + overwrite_reboot_timeout(1); + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + sleep(TWO_SECONDS); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT(response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "failed to warm reboot")); + +} + TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) { sigterm_requested = true; set_mock_defaults(); @@ -331,6 +409,25 @@ TEST_P(RebootBEAutoStartTest, TestColdBootSigterm) { IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_P(RebootBEAutoStartTest, TestWarmBootSigterm) { + sigterm_requested = true; + set_mock_defaults(); + overwrite_reboot_timeout(1); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + sleep(ONE_SECOND); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc(); + EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT( + second_resp, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) { // Return FAIL from dbus reboot call. DbusInterface::DbusResponse dbus_response{ @@ -353,6 +450,28 @@ TEST_P(RebootBEAutoStartTest, TestColdBootDbusError) { "dbus reboot failed")); } +TEST_P(RebootBEAutoStartTest, TestWarmBootDbusError) { + // Return FAIL from dbus reboot call. + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_FAIL, "dbus reboot failed"}; + EXPECT_CALL(m_dbus_interface, Reboot(_)) + .Times(1) + .WillOnce(Return(dbus_response)); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + sleep(TWO_SECONDS); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + gnoi::system::RebootStatusResponse second_resp = do_reboot_status_rpc(); + EXPECT_THAT(second_resp, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT(second_resp, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "dbus reboot failed")); +} + TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) { set_mock_defaults(); @@ -374,12 +493,66 @@ TEST_P(RebootBEAutoStartTest, TestStopDuringColdBoot) { IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); } +TEST_P(RebootBEAutoStartTest, TestStopDuringWarmBoot) { + set_mock_defaults(); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); +// std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + send_stop_reboot_thread(); + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + EXPECT_THAT( + response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_UNKNOWN, "")); +} + TEST_P(RebootBEAutoStartTest, TestInvalidJsonRebootRequest) { std::string json_request = "abcd"; NotificationResponse response = handle_reboot_request(json_request); EXPECT_EQ(swss::StatusCode::SWSS_RC_INTERNAL, response.status); } +TEST_P(RebootBEAutoStartTest, TestWarmFailureFollowedByColdBoot) { + DbusInterface::DbusResponse dbus_response{ + DbusInterface::DbusStatus::DBUS_SUCCESS, ""}; + overwrite_reboot_timeout(1); + + RebootRequest request; + request.set_method(RebootMethod::WARM); + start_reboot_via_rpc(request); + + std::this_thread::sleep_for(std::chrono::milliseconds(TENTH_SECOND_MS)); + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), + RebootBE::RebManagerStatus::WARM_REBOOT_IN_PROGRESS); + + std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS)); + gnoi::system::RebootStatusResponse response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 1, RebootMethod::WARM)); + + + request.set_method(RebootMethod::COLD); + start_reboot_via_rpc(request); + + // We have to wait for the 1 second reboot Timeout + std::this_thread::sleep_for(std::chrono::milliseconds(TWO_SECONDS_MS)); + + EXPECT_EQ(m_rebootbe.GetCurrentStatus(), RebootBE::RebManagerStatus::IDLE); + response = do_reboot_status_rpc(); + EXPECT_THAT(response, ActiveCountMethod(false, 2, RebootMethod::COLD)); + EXPECT_THAT(response, + IsStatus(RebootStatus_Status::RebootStatus_Status_STATUS_FAILURE, + "platform failed to reboot")); +} + + INSTANTIATE_TEST_SUITE_P(TestWithStartupWarmbootEnabledState, RebootBEAutoStartTest, testing::Values(true, false));