Skip to content

Commit

Permalink
added configurable retryable status codes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jformacek
jformacek committed Jan 9, 2025
1 parent 161464f commit 9fe5270
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 14 deletions.
31 changes: 25 additions & 6 deletions Commands/Public/Invoke-ExoCommand.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
#See also https://makolyte.com/csharp-how-to-change-the-httpclient-timeout-per-request/ for more details on timeouts of http client
$Timeout,

[Parameter()]
#Status codes that are considered retryable
[System.Net.HttpStatusCode[]]$RetryableStatusCodes = @([System.Net.HttpStatusCode]::TooManyRequests),

[switch]
#If we want to write any warnings returned by EXO REST API
$ShowWarnings,
Expand Down Expand Up @@ -173,7 +177,7 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
$responseData = $Payload | ConvertFrom-Json
}
catch {
Write-Warning "Received unexpected non-JSON response: $payload"
Write-Warning "Received unexpected non-JSON response with http staus $($response.StatusCode): $payload"
$responseData = $payload
}
}
Expand Down Expand Up @@ -224,17 +228,18 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
{
#request failed
$ex = $null
if($response.StatusCode -ne [System.Net.HttpStatusCode]::TooManyRequests)
if($response.StatusCode -notin $RetryableStatusCodes -and $responseData -notlike 'You have reached the maximum number of concurrent requests per tenant. Please wait and try again*')
{
$shouldContinue = $false
$ex = $responseData | Get-ExoException -httpCode $response.StatusCode
}
else
{
#we wait on http 429 or throttling message
if($retries -ge $MaxRetries)
{
$shouldContinue = $false
$ex = New-Object ExoHelper.ExoException($response.StatusCode, 'ExoTooManyRequests', '', 'Max retry count for throttled request exceeded')
$ex = New-Object ExoHelper.ExoException($response.StatusCode, 'ExoTooManyRetries', '', 'Max retry count for throttled request exceeded')
}
}
if($null -ne $ex)
Expand All @@ -247,15 +252,29 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
}
}
#TooManyRequests --> let's wait and retry
$headers = @{}
$response.Headers | ForEach-Object { $headers[$_.Key] = $_.Value }
$headersObject = [PSCustomObject]$headers

$retries++
switch($WarningPreference)
{
'Continue' { Write-Warning "Retry #$retries" }
'SilentlyContinue' { Write-Verbose "Retry #$retries" }
'Continue' {
Write-Warning "Retry #$retries"
$headersObject | Out-String | Write-Warning
break;
}
'SilentlyContinue' {
Write-Verbose "Retry #$retries"
$headersObject | Out-String | Write-Verbose
break;
}
}

#wait some time
Start-Sleep -Seconds $retries
$val = $retries
#if($response.Headers.TryGetValues('Retry-After', [ref]$val))
Start-Sleep -Seconds ($retries * 5)
}
}
catch
Expand Down
2 changes: 1 addition & 1 deletion Module/ExoHelper/ExoHelper.psd1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ PrivateData = @{
# ReleaseNotes = ''

# Prerelease string of this module
Prerelease = 'beta2'
Prerelease = 'beta3'

# Flag to indicate whether the module requires explicit user acceptance for install/update/save
# RequireLicenseAcceptance = $false
Expand Down
32 changes: 25 additions & 7 deletions Module/ExoHelper/ExoHelper.psm1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
#See also https://makolyte.com/csharp-how-to-change-the-httpclient-timeout-per-request/ for more details on timeouts of http client
$Timeout,

[Parameter()]
#Status codes that are considered retryable
[System.Net.HttpStatusCode[]]$RetryableStatusCodes = @([System.Net.HttpStatusCode]::TooManyRequests),

[switch]
#If we want to write any warnings returned by EXO REST API
$ShowWarnings,
Expand Down Expand Up @@ -233,13 +237,13 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
$responseData = $Payload | ConvertFrom-Json
}
catch {
Write-Warning "Received unexpected non-JSON response: $payload"
Write-Warning "Received unexpected non-JSON response with http staus $($response.StatusCode): $payload"
$responseData = $payload
}
}
else
{
Write-Warning "Received non-JSON response: $($response.content.Headers.ContentType.MediaType)"
Write-Warning "Received non-JSON response with http status $($response.StatusCode): $($response.content.Headers.ContentType.MediaType)"
$responseData = $payload
}
}
Expand Down Expand Up @@ -284,13 +288,14 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
{
#request failed
$ex = $null
if($response.StatusCode -ne [System.Net.HttpStatusCode]::TooManyRequests)
if($response.StatusCode -notin $RetryableStatusCodes -and $responseData -notlike 'You have reached the maximum number of concurrent requests per tenant. Please wait and try again*')
{
$shouldContinue = $false
$ex = $responseData | Get-ExoException -httpCode $response.StatusCode
}
else
{
#we wait on http 429 or throttling message
if($retries -ge $MaxRetries)
{
$shouldContinue = $false
Expand All @@ -299,7 +304,6 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
}
if($null -ne $ex)
{
Write-Verbose "Handling exception: StatusCode $($ex.StatusCode)`tExoErrorCode: $($ex.ExoErrorCode)`tExoErrorType: $($ex.ExoErrorType)"
switch($ErrorActionPreference)
{
'Stop' { throw $ex }
Expand All @@ -308,15 +312,29 @@ This command creates connection for IPPS REST API, retrieves list of sensitivity
}
}
#TooManyRequests --> let's wait and retry
$headers = @{}
$response.Headers | ForEach-Object { $headers[$_.Key] = $_.Value }
$headersObject = [PSCustomObject]$headers

$retries++
switch($WarningPreference)
{
'Continue' { Write-Warning "Retry #$retries" }
'SilentlyContinue' { Write-Verbose "Retry #$retries" }
'Continue' {
Write-Warning "Retry #$retries"
$headersObject | Out-String | Write-Warning
break;
}
'SilentlyContinue' {
Write-Verbose "Retry #$retries"
$headersObject | Out-String | Write-Verbose
break;
}
}

#wait some time
Start-Sleep -Seconds $retries
$val = $retries
#if($response.Headers.TryGetValues('Retry-After', [ref]$val))
Start-Sleep -Seconds ($retries * 5)
}
}
catch
Expand Down
1 change: 1 addition & 0 deletions Module/ExoHelper/Helpers/ExoException.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public class ExoException : Exception
public string ExoErrorType { get; set; }
public ExoException(HttpStatusCode? statusCode, string exoCode, string exoErrorType, string message):this(statusCode, exoCode, exoErrorType, message, null)
{

}
public ExoException(HttpStatusCode? statusCode, string exoCode, string exoErrorType, string message, Exception innerException):base(message, innerException)
{
Expand Down

0 comments on commit 9fe5270

Please sign in to comment.