func: add mitigation for deployments that take awhile to "come alive" (#4449)

Add polling delay and retries for function app consumption deployments that take awhile to "come alive"

Fixes #4439

Author: weikanglim

cli/azd/pkg/azsdk/funcapp_host_client.go

@@ -117,9 +117,12 @@ func (c *FuncAppHostClient) Publish(
 
 func (c *FuncAppHostClient) waitForDeployment(ctx context.Context, location string) (PublishResponse, error) {
 	// This frequency is recommended by the service team.
-	polLDelay := 1 * time.Second
+	pollDelay := 1 * time.Second
 	var lastResponse *PublishResponse
 
+	// mitigation for deployments that take awhile to "come alive"
+	deploymentNotFoundAttempts := 0
+
 	for {
 		req, err := runtime.NewRequest(ctx, http.MethodGet, location)
 		if err != nil {
@@ -131,6 +134,12 @@ func (c *FuncAppHostClient) waitForDeployment(ctx context.Context, location stri
 			return PublishResponse{}, err
 		}
 
+		if deploymentNotFoundAttempts <= 3 && response.StatusCode == http.StatusNotFound {
+			deploymentNotFoundAttempts++
+			time.Sleep(pollDelay)
+			continue
+		}
+
 		// It's possible to observe a 404 response after the deployment is complete.
 		// If a 404 is observed, we assume the deployment is complete and return the last response.
 		//
@@ -167,9 +176,9 @@ func (c *FuncAppHostClient) waitForDeployment(ctx context.Context, location stri
 		// Record the latest response
 		lastResponse = &resp
 
-		delay := polLDelay
+		delay := pollDelay
 		if retryAfter := httputil.RetryAfter(response); retryAfter > 0 {
-			delay = polLDelay
+			delay = pollDelay
 		}
 
 		select {