Port infinite re-ATTACH bug fix to LTS 01/2023 (#2474)

* Port 1f4fd95 to lts_01_2023

* Port a92ee04 to lts_01_2023 (except ASan)

* Remove cleanup steps in vsts.yml

* Port 89db361 to lts_01_2023

build/.vsts-ci.yml

@@ -62,10 +62,10 @@ jobs:
   displayName: "Windows x86"
   steps:
   - script: |
-      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat"
       call jenkins\windows_c.cmd
     displayName: 'Build'
   - script: |
+      call jenkins\windows_c_VsDevCmd.cmd x86
       cd cmake && ctest -T test --no-compress-output -C "Debug" -V -j 16 --schedule-random
     displayName: "Run Tests"
     env:
@@ -99,11 +99,11 @@ jobs:
   displayName: 'Windows x64 (Debug)'
   steps:
   - script: |
-      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat"
       call jenkins\windows_c.cmd --platform x64
     displayName: 'Build'
   - script: |
-        cd cmake && ctest -T test --no-compress-output -C "Debug" -V -j 16 --schedule-random
+      call jenkins\windows_c_VsDevCmd.cmd x64
+      cd cmake && ctest -T test --no-compress-output -C "Debug" -V -j 16 --schedule-random
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -139,11 +139,11 @@ jobs:
   displayName: 'Windows x64 (Release)'
   steps:
   - script: |
-      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat"
       call jenkins\windows_c_release.cmd --platform x64
     displayName: 'Build'
   - script: |
-        cd cmake && ctest -T test --no-compress-output -C "Release" -V -j 16 --schedule-random
+      call jenkins\windows_c_VsDevCmd.cmd x64
+      cd cmake && ctest -T test --no-compress-output -C "Release" -V -j 16 --schedule-random
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -179,7 +179,7 @@ jobs:
   displayName: "Windows Dynamic"
   steps:
   - script: |
-      call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Enterprise\Common7\Tools\VsDevCmd.bat"
+      call jenkins\windows_c_VsDevCmd.cmd
       call jenkins\windows_c_build_as_dynamic.cmd
     displayName: 'Build'
     env:
@@ -201,9 +201,6 @@ jobs:
       IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64)
       IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
       IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID)
-  - script: cd .. && rd /Q /S $(Agent.BuildDirectory)\s
-    displayName: 'Cleanup'
-    condition: always()
 - job: linuxoptions
   container: linux-c-ubuntu-2004
   pool: 
@@ -829,9 +826,6 @@ jobs:
       mergeTestResults: true
       testRunTitle: 'OSX'
     condition: succeededOrFailed()
-  - script: rm -rf $(Agent.BuildDirectory)/*
-    displayName: 'Cleanup'
-    condition: always()
 - job: xcodenative
   pool:
     name: OSX
@@ -873,9 +867,6 @@ jobs:
       mergeTestResults: true
       testRunTitle: 'XCode'
     condition: succeededOrFailed()
-  - script: rm -rf $(Agent.BuildDirectory)/*
-    displayName: 'Cleanup'
-    condition: always()
 - job: raspberrypi
   container: raspberrypi-c-buster
   pool: 

build_all/windows/build.cmd

@@ -146,28 +146,25 @@ mkdir %cmake-root%\cmake
 rem no error checking
 pushd %cmake-root%\cmake
 
+call %build-root%\jenkins\windows_c_VsDevCmd.cmd %build-platform%
+
+where cmake
+cmake --version
+
 if %build-platform% == x64 (
     echo ***Running CMAKE for Win64***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Duse_cppunittest:BOOL=OFF -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G "Visual Studio 15 2017" -A x64 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=$build_config
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Duse_cppunittest:BOOL=OFF -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A x64 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 ) else if %build-platform% == arm (
     echo ***Running CMAKE for ARM***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G "Visual Studio 15 2017" -A ARM  -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=$build_config
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A ARM -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 ) else (
     echo ***Running CMAKE for Win32***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G "Visual Studio 15 2017" -A Win32 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=$build_config
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A Win32 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 )
 
-echo ***checking msbuild***
-where /q msbuild
-IF ERRORLEVEL 1 (
-echo ***setting VC paths***
-    IF EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Enterprise\Common7\Tools\VsMSBuildCmd.bat" call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Enterprise\Common7\Tools\VsMSBuildCmd.bat"
-)
-where msbuild
-
 if %make%==yes (
     echo build-config is %build-config%
 

iothub_client/src/iothubtransport_amqp_common.c

@@ -157,6 +157,7 @@ typedef struct AMQP_TRANSPORT_DEVICE_TWIN_CONTEXT_TAG
     uint32_t item_id;
     TRANSPORT_CALLBACKS_INFO transport_callbacks;
     void* transport_ctx;
+    AMQP_TRANSPORT_DEVICE_INSTANCE* device;
 } AMQP_TRANSPORT_DEVICE_TWIN_CONTEXT;
 
 typedef struct AMQP_TRANSPORT_GET_TWIN_CONTEXT_TAG
@@ -442,6 +443,8 @@ static DEVICE_MESSAGE_DISPOSITION_RESULT on_message_received(IOTHUB_MESSAGE_HAND
         {
             device_disposition_result = DEVICE_MESSAGE_DISPOSITION_RESULT_NONE;
         }
+
+        amqp_device_instance->number_of_previous_failures = 0;
     }
 
     return device_disposition_result;
@@ -472,6 +475,7 @@ static int on_method_request_received(void* context, const char* method_name, co
     }
     else
     {
+        device_state->number_of_previous_failures = 0;
         result = 0;
     }
     return result;
@@ -520,9 +524,13 @@ static void on_device_send_twin_update_complete_callback(DEVICE_TWIN_UPDATE_RESU
     else
     {
         AMQP_TRANSPORT_DEVICE_TWIN_CONTEXT* dev_twin_ctx = (AMQP_TRANSPORT_DEVICE_TWIN_CONTEXT*)context;
-
         dev_twin_ctx->transport_callbacks.twin_rpt_state_complete_cb(dev_twin_ctx->item_id, status_code, dev_twin_ctx->transport_ctx);
 
+        if (status_code >= 200 && status_code < 300)
+        {
+            dev_twin_ctx->device->number_of_previous_failures = 0;
+        }
+
         free(dev_twin_ctx);
     }
 }
@@ -539,6 +547,8 @@ static void on_device_twin_update_received_callback(DEVICE_TWIN_UPDATE_TYPE upda
 
         registered_device->transport_instance->transport_callbacks.twin_retrieve_prop_complete_cb((update_type == DEVICE_TWIN_UPDATE_TYPE_COMPLETE ? DEVICE_TWIN_UPDATE_COMPLETE : DEVICE_TWIN_UPDATE_PARTIAL),
                 message, length, registered_device->transport_instance->transport_ctx);
+
+        registered_device->number_of_previous_failures = 0;
     }
 }
 
@@ -948,6 +958,7 @@ static void on_event_send_complete(IOTHUB_MESSAGE_LIST* message, D2C_EVENT_SEND_
     else
     {
         registered_device->number_of_send_event_complete_failures = 0;
+        registered_device->number_of_previous_failures = 0;
     }
 
     if (result == D2C_EVENT_SEND_COMPLETE_RESULT_ERROR_QUOTA_EXCEEDED)
@@ -1090,7 +1101,6 @@ static int IoTHubTransport_AMQP_Common_Device_DoWork(AMQP_TRANSPORT_DEVICE_INSTA
         }
         else
         {
-            registered_device->number_of_previous_failures = 0;
             result = RESULT_OK;
         }
     }
@@ -1364,6 +1374,7 @@ IOTHUB_PROCESS_ITEM_RESULT IoTHubTransport_AMQP_Common_ProcessItem(TRANSPORT_LL_
                 dev_twin_ctx->transport_callbacks = transport_instance->transport_callbacks;
                 dev_twin_ctx->transport_ctx = transport_instance->transport_ctx;
                 dev_twin_ctx->item_id = iothub_item->device_twin->item_id;
+                dev_twin_ctx->device = registered_device;
 
                 if (amqp_device_send_twin_update_async(
                     registered_device->device_handle,

iothub_client/tests/common_e2e/iothubclient_common_e2e.c

@@ -204,7 +204,7 @@ static int IoTHubCallback(void* context, const char* data, size_t size)
 // Invoked when a connection status changes.  Tests poll the status in the connection_status_info to make sure expected transitions occur.
 static void connection_status_callback(IOTHUB_CLIENT_CONNECTION_STATUS status, IOTHUB_CLIENT_CONNECTION_STATUS_REASON reason, void* userContextCallback)
 {
-    LogInfo("connection_status_callback: status=<%d>, reason=<%s>", status, MU_ENUM_TO_STRING(IOTHUB_CLIENT_CONNECTION_STATUS_REASON, reason));
+    LogInfo("connection_status_callback: status=<%s>, reason=<%s>", MU_ENUM_TO_STRING(IOTHUB_CLIENT_CONNECTION_STATUS, status), MU_ENUM_TO_STRING(IOTHUB_CLIENT_CONNECTION_STATUS_REASON, reason));
 
     CONNECTION_STATUS_INFO* connection_status_info = (CONNECTION_STATUS_INFO*)userContextCallback;
     ASSERT_ARE_EQUAL(LOCK_RESULT, LOCK_OK, Lock(connection_status_info->lock));
@@ -1622,7 +1622,7 @@ static void recv_message_test(IOTHUB_PROVISIONED_DEVICE* deviceToUse, IOTHUB_CLI
     client_connect_to_hub(deviceToUse, protocol);
 
     // Make sure we have a connection
-    ASSERT_IS_TRUE(client_wait_for_connection_restored(), "Connection Callback has not been called");
+    ASSERT_IS_TRUE(wait_for_client_authenticated(client_conn_wait_time), "Connection Callback has not been called");
 
     // Create receive context
     const char* msg_content;

iothub_client/tests/iothub_client_properties_ut/CMakeLists.txt

@@ -2,7 +2,7 @@
 #Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 #this is CMakeLists.txt for version
-cmake_minimum_required(VERSION 2.8.11)
+cmake_minimum_required(VERSION 3.5)
 
 compileAsC99()
 set(theseTestsName iothub_client_properties_ut)

iothub_client/tests/iothubtransport_amqp_common_ut/iothubtransport_amqp_common_ut.c

@@ -473,7 +473,8 @@ static void on_umock_c_error(UMOCK_C_ERROR_CODE error_code)
 #define TEST_X509_PRIVATE_KEY                      "Raphael Rabello"
 #define TEST_MESSAGE_SOURCE_CHAR_PTR               "messagereceiver_link_name"
 #define TEST_RETRY_CONTROL_HANDLE                  (RETRY_CONTROL_HANDLE)0x4276
-
+// This parameter must match the namesake macro in iothubtransport_amqp_common.c
+#define MAX_NUMBER_OF_DEVICE_FAILURES              5
 static TRANSPORT_CALLBACKS_INFO transport_cb_info;
 static void* transport_cb_ctx = (void*)0x499922;
 
@@ -488,6 +489,25 @@ static DLIST_ENTRY TEST_waitingToSend;
 static unsigned long TEST_MESSAGE_ID;
 
 
+// ---------- Time-related Test Helpers ---------- //
+static time_t add_seconds(time_t base_time, unsigned int seconds)
+{
+    time_t new_time;
+    struct tm* bd_new_time;
+
+    if ((bd_new_time = localtime(&base_time)) == NULL)
+    {
+        new_time = INDEFINITE_TIME;
+    }
+    else
+    {
+        bd_new_time->tm_sec += seconds;
+        new_time = mktime(bd_new_time);
+    }
+
+    return new_time;
+}
+
 // ---------- Helpers for Expected Calls ---------- //
 
 // @remarks
@@ -4329,6 +4349,137 @@ TEST_FUNCTION(ConnectionStatusCallBack_UNAUTH_msg_communication_error)
     destroy_transport(handle, device_handle, NULL);
 }
 
+TEST_FUNCTION(Transport_Fully_Reconnects_After_MAX_NUMBER_OF_DEVICE_FAILURES_AMQP_device_errors)
+{
+    // arrange
+    initialize_test_variables();
+    time_t current_time = TEST_current_time;
+    bool is_state_change_timedout = false;
+    TRANSPORT_LL_HANDLE handle = create_transport();
+    IOTHUB_DEVICE_CONFIG* device_config = create_device_config(TEST_DEVICE_ID_CHAR_PTR, true);
+
+    IOTHUB_DEVICE_HANDLE device_handle;
+    device_handle = register_device(handle, device_config, &TEST_waitingToSend, true);
+
+    crank_transport_ready_after_create(handle, &TEST_waitingToSend, 0, false, true, 1, TEST_current_time, false);
+
+    // act
+    // Fail AMQP device enough times to trigger reconnection:
+    umock_c_reset_all_calls();
+
+    for (int i = 0; i < (MAX_NUMBER_OF_DEVICE_FAILURES - 1); i++)
+    {
+        // Raise error from amqp_device level.
+        STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+        STRICT_EXPECTED_CALL(Transport_ConnectionStatusCallBack(IOTHUB_CLIENT_CONNECTION_UNAUTHENTICATED, IOTHUB_CLIENT_CONNECTION_COMMUNICATION_ERROR, IGNORED_PTR_ARG));
+        TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+            DEVICE_STATE_STARTED, DEVICE_STATE_ERROR_MSG);
+
+        // On first error, restablish the links (amqp_device_delayed_stop)
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_device_delayed_stop(IGNORED_PTR_ARG, IGNORED_NUM_ARG));
+        STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+        (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+        STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+        TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+            DEVICE_STATE_ERROR_MSG, DEVICE_STATE_STOPPING);
+
+        // amqp_device_delayed_stop takes 10 seconds to fully stop.
+        current_time = add_seconds(current_time, 10);
+
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(is_timeout_reached(IGNORED_NUM_ARG, IGNORED_NUM_ARG, IGNORED_PTR_ARG))
+            .CopyOutArgumentBuffer_is_timed_out(&is_state_change_timedout, sizeof(is_state_change_timedout))
+            .SetReturn(0);
+        STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+        (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+        // Complete stopping amqp_device.
+        STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+        STRICT_EXPECTED_CALL(Transport_ConnectionStatusCallBack(
+            IOTHUB_CLIENT_CONNECTION_UNAUTHENTICATED, IOTHUB_CLIENT_CONNECTION_OK, IGNORED_PTR_ARG));
+        TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+            DEVICE_STATE_STOPPING, DEVICE_STATE_STOPPED);
+
+        current_time = add_seconds(current_time, 1);
+
+        // AMQP transport starts the amqp_device again.
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_get_session_handle(IGNORED_PTR_ARG, IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_get_cbs_handle(IGNORED_PTR_ARG, IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_device_start_async(IGNORED_PTR_ARG, IGNORED_PTR_ARG, IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+        (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+        STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+        TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+            DEVICE_STATE_STOPPED, DEVICE_STATE_STARTING);
+
+        // amqp_device now goes fully started.
+        current_time = add_seconds(current_time, 1);
+
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(is_timeout_reached(IGNORED_NUM_ARG, IGNORED_NUM_ARG, IGNORED_PTR_ARG))
+            .CopyOutArgumentBuffer_is_timed_out(&is_state_change_timedout, sizeof(is_state_change_timedout))
+            .SetReturn(0);
+        STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+        (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+        STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+        STRICT_EXPECTED_CALL(retry_control_reset(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(Transport_ConnectionStatusCallBack(
+            IOTHUB_CLIENT_CONNECTION_AUTHENTICATED, IOTHUB_CLIENT_CONNECTION_OK, IGNORED_PTR_ARG));
+        TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+            DEVICE_STATE_STARTING, DEVICE_STATE_STARTED);
+
+        // Regular DoWork call with no issues.
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(DList_IsListEmpty(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+        STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+        (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+    }
+
+    // Raise 5th error from amqp_device, triggering AMQP transport to do a full reconnection.
+    STRICT_EXPECTED_CALL(get_time(NULL)).SetReturn(current_time);
+    STRICT_EXPECTED_CALL(Transport_ConnectionStatusCallBack(IOTHUB_CLIENT_CONNECTION_UNAUTHENTICATED, IOTHUB_CLIENT_CONNECTION_COMMUNICATION_ERROR, IGNORED_PTR_ARG));
+    TEST_device_create_saved_on_state_changed_callback(TEST_device_create_saved_on_state_changed_context,
+        DEVICE_STATE_STARTED, DEVICE_STATE_ERROR_MSG);
+
+    STRICT_EXPECTED_CALL(singlylinkedlist_get_head_item(IGNORED_PTR_ARG));
+    STRICT_EXPECTED_CALL(singlylinkedlist_item_get_value(IGNORED_PTR_ARG));
+    STRICT_EXPECTED_CALL(STRING_c_str(IGNORED_PTR_ARG));
+    STRICT_EXPECTED_CALL(amqp_device_do_work(IGNORED_PTR_ARG));
+    STRICT_EXPECTED_CALL(singlylinkedlist_get_next_item(IGNORED_PTR_ARG));
+    STRICT_EXPECTED_CALL(amqp_connection_do_work(IGNORED_PTR_ARG));
+    (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+    // Observe the AMQP transport do a full re-connection.
+    set_expected_calls_for_prepare_for_connection_retry(1, DEVICE_STATE_ERROR_MSG);
+    (void)IoTHubTransport_AMQP_Common_DoWork(handle);
+
+    // assert
+    ASSERT_ARE_EQUAL(char_ptr, umock_c_get_expected_calls(), umock_c_get_actual_calls());
+
+    // cleanup
+    destroy_transport(handle, device_handle, NULL);
+}
+
 TEST_FUNCTION(ConnectionStatusCallBack_UNAUTH_no_network)
 {
     // arrange