From b9971c633c0695068aabf22a402293e38031fe60 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Wed, 15 May 2024 20:19:37 +0200 Subject: [PATCH 01/20] 6.4.6 --- .vscode/settings.json | 5 +- README.md | 23 +- history.md | 5 + pwsh/AzGovVizParallel.ps1 | 210 ++++++++++-------- pwsh/dev/buildAzGovVizParallel.ps1 | 4 +- pwsh/dev/devAzGovVizParallel.ps1 | 124 +++++------ pwsh/dev/functions/addHtParameters.ps1 | 2 +- pwsh/dev/functions/addIndexNumberToArray.ps1 | 2 +- pwsh/dev/functions/addRowToTable.ps1 | 2 +- pwsh/dev/functions/apiCallTracking.ps1 | 2 +- pwsh/dev/functions/buildJSON.ps1 | 2 +- pwsh/dev/functions/buildMD.ps1 | 2 +- pwsh/dev/functions/buildPolicyAllJSON.ps1 | 2 +- pwsh/dev/functions/buildTree.ps1 | 2 +- pwsh/dev/functions/cacheBuiltIn.ps1 | 2 +- pwsh/dev/functions/checkAzGovVizVersion.ps1 | 2 +- pwsh/dev/functions/createTagList.ps1 | 2 +- .../dataCollectionFunctions.ps1 | 12 +- pwsh/dev/functions/detailSubscriptions.ps1 | 2 +- pwsh/dev/functions/detectPolicyEffect.ps1 | 2 +- pwsh/dev/functions/exportBaseCSV.ps1 | 2 +- pwsh/dev/functions/exportResourceLocks.ps1 | 2 +- pwsh/dev/functions/getConsumption.ps1 | 2 +- .../functions/getDefaultManagementGroup.ps1 | 2 +- pwsh/dev/functions/getEntities.ps1 | 7 +- pwsh/dev/functions/getFileNaming.ps1 | 2 +- pwsh/dev/functions/getGroupmembers.ps1 | 2 +- pwsh/dev/functions/getMDfCSecureScoreMG.ps1 | 2 +- pwsh/dev/functions/getOrphanedResources.ps1 | 2 +- pwsh/dev/functions/getPIMEligible.ps1 | 18 +- pwsh/dev/functions/getPolicyHash.ps1 | 5 +- pwsh/dev/functions/getPolicyRemediation.ps1 | 2 +- ...getPrivateEndpointCapableResourceTypes.ps1 | 2 +- .../getResourceDiagnosticsCapability.ps1 | 2 +- pwsh/dev/functions/getSubscriptions.ps1 | 2 +- pwsh/dev/functions/getTenantDetails.ps1 | 2 +- pwsh/dev/functions/handleCloudEnvironment.ps1 | 2 +- pwsh/dev/functions/html/htmlFunctions.ps1 | 2 +- pwsh/dev/functions/namingValidation.ps1 | 2 +- pwsh/dev/functions/prepareData.ps1 | 2 +- pwsh/dev/functions/processAADGroups.ps1 | 2 +- .../processALZPolicyVersionChecker.ps1 | 11 +- pwsh/dev/functions/processApplications.ps1 | 2 +- pwsh/dev/functions/processDataCollection.ps1 | 2 +- .../functions/processDefinitionInsights.ps1 | 8 +- pwsh/dev/functions/processDiagramMermaid.ps1 | 2 +- .../dev/functions/processHierarchyMapOnly.ps1 | 2 +- .../processHierarchyMapOnlyCustomData.ps1 | 2 +- .../functions/processManagedIdentities.ps1 | 2 +- pwsh/dev/functions/processNetwork.ps1 | 2 +- .../dev/functions/processPrivateEndpoints.ps1 | 2 +- .../functions/processScopeInsightsMgOrSub.ps1 | 8 +- .../processStorageAccountAnalysis.ps1 | 2 +- pwsh/dev/functions/processTenantSummary.ps1 | 21 +- .../functions/removeInvalidFileNameChars.ps1 | 12 +- pwsh/dev/functions/resolveObjectIds.ps1 | 2 +- pwsh/dev/functions/runInfo.ps1 | 2 +- pwsh/dev/functions/selectMg.ps1 | 2 +- pwsh/dev/functions/setBaseVariablesMG.ps1 | 2 +- pwsh/dev/functions/setOutput.ps1 | 2 +- pwsh/dev/functions/setTranscript.ps1 | 2 +- pwsh/dev/functions/showMemoryUsage.ps1 | 2 +- pwsh/dev/functions/stats.ps1 | 2 +- pwsh/dev/functions/testGuid.ps1 | 2 +- pwsh/dev/functions/validateAccess.ps1 | 2 +- .../validateLeastPrivilegeForUser.ps1 | 4 +- pwsh/dev/functions/verifyModules3rd.ps1 | 2 +- pwsh/prerequisites.ps1 | 2 +- version.json | 2 +- 69 files changed, 302 insertions(+), 281 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 57005a66..3c683774 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,5 +20,8 @@ "powershell.codeFormatting.whitespaceBeforeOpenBrace": true, "powershell.codeFormatting.whitespaceBeforeOpenParen": true, "powershell.codeFormatting.whitespaceBetweenParameters": true, - "markdown.extension.toc.unorderedList.marker": "*" + "markdown.extension.toc.unorderedList.marker": "*", + "[powershell]": { + "files.encoding": "utf8bom" + } } \ No newline at end of file diff --git a/README.md b/README.md index dde85fa7..851cc7da 100644 --- a/README.md +++ b/README.md @@ -87,25 +87,10 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-May-05 / 6.4.5 Minor) - -- updated orphaned resources queries following the source repository [Azure Orphan Resources - GitHub](https://github.com/dolevshor/azure-orphan-resources/blob/111a7ea4ced2016760b1b95544f298b9b4be8dee/Queries/orphan-resources-queries.md) with slight adjustments -- covering _I´ll call it_ 'tenant/service level Role definitions' -- optimize/bug fix 'Processing roleDefinitions used in policyDefinitions' -- increase the default value for `-AzureConsumptionPeriod` from `1` to `2` - if the Azure Governance Visualizer is executed early in the day, consumption data may not be accurate enough.. (reminder: the switch parameter `-DoAzureConsumption` must be set to `true` for the consumption data collection to kick in) -- update default value for parameter `-ValidPolicyEffects` -- update [API reference](#api-reference) Microsoft.Authorization/roleDefinitions use API version 2023-07-01-preview (previous 2022-05-01-preview) -- update [API reference](#api-reference) Microsoft.ResourceGraph/resources use API version 2022-10-01 (previous 2021-03-01) -- update [API reference](#api-reference) Microsoft.CostManagement/query use API version 2024-01-01 (previous 2023-03-01) - -**Changes** (2024-Apr-17 / 6.4.4 Minor) - -- fix issue #230 - - use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.1 -- update [API reference](#api-reference) Microsoft.Security/pricings use API version 2024-01-01 (previous 2018-06-01) -- add 'Mutate' to `ValidPolicyEffects` -- location related tasks - use only physical locations (exclude logical) -- optimize collection of Role definitions that are used in Policy definitions +**Changes** (2024-May-15 / 6.4.6 Minor) + +- DevSkim and PSScriptAnalyzer integration +- fixes and optimization based on DevSkim and PSScriptAnalyzer findings [Full release history](history.md) diff --git a/history.md b/history.md index 4bb4d1c9..3bfd792a 100644 --- a/history.md +++ b/history.md @@ -4,6 +4,11 @@ ### Azure Governance Visualizer version 6 +**Changes** (2024-May-15 / 6.4.6 Minor) + +- DevSkim and PSScriptAnalyzer integration +- fixes and optimization based on DevSkim and PSScriptAnalyzer findings + **Changes** (2024-May-05 / 6.4.5 Minor) - updated orphaned resources queries following the source repository [Azure Orphan Resources - GitHub](https://github.com/dolevshor/azure-orphan-resources/blob/111a7ea4ced2016760b1b95544f298b9b4be8dee/Queries/orphan-resources-queries.md) with slight adjustments diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 50d173b7..b2a6665f 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -1,4 +1,4 @@ -<# +<# .SYNOPSIS This script creates the following files to help better understand and audit your governance setup csv file @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.5', + $ProductVersion = '6.4.6', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -403,7 +403,7 @@ Param $NoCsvExport, [string] - [parameter(ValueFromPipeline)][ValidateSet(';', ',')][string]$CsvDelimiter = ';', + [ValidateSet(';', ',')]$CsvDelimiter = ';', [switch] $CsvExportUseQuotesAsNeeded, @@ -483,7 +483,7 @@ Param $DoNotIncludeResourceGroupsAndResourcesOnRBAC, [Alias('AzureDevOpsWikiHierarchyDirection')] - [parameter(ValueFromPipeline)][ValidateSet('TD', 'LR')][string]$MermaidDirection = 'TD', + [ValidateSet('TD', 'LR')][string]$MermaidDirection = 'TD', [int] $ChangeTrackingDays = 14, @@ -639,7 +639,7 @@ Set-Item Env:\SuppressAzurePowerShellBreakingChangeWarnings 'true' #start $startAzGovViz = Get-Date $startTime = Get-Date -Format 'dd-MMM-yyyy HH:mm:ss' -Write-Host "Start Azure Governance Visualizer $($startTime) (#$($ProductVersion))" +Write-Host "Start Azure Governance Visualizer (aka $Product) $($startTime) (#$($ProductVersion))" if ($ManagementGroupId -match ' ') { Write-Host "Provided Management Group ID: '$($ManagementGroupId)'" -ForegroundColor Yellow @@ -3390,11 +3390,12 @@ function getEntities { $array = $entity.properties.parentNameChain $array += $entity.name + $script:htSubscriptionsMgPath.($entity.name) = @{ ParentNameChain = $entity.properties.parentNameChain ParentNameChainDelimited = $entity.properties.parentNameChain -join '/' - Parent = $entity.properties.parent.Id -replace '.*/' - ParentName = $htEntitiesPlain.($entity.properties.parent.Id -replace '.*/').properties.displayName + Parent = $parent + ParentName = $htEntitiesPlain.($parent).properties.displayName DisplayName = $entity.properties.displayName path = $array pathDelimited = $array -join '/' @@ -4194,15 +4195,16 @@ function getPIMEligible { if (-not $PIMEligibilityIgnoreScope) { if (($azAPICallConf['checkContext']).Tenant.Id -ne $ManagementGroupId) { foreach ($entry in $res) { + $entryIdGuid = $entry.externalId -replace '.*/' if ($entry.type -eq 'managementGroup') { - if ($htManagementGroupsMgPath.($ManagementGroupId).ParentNameChain -contains ($entry.externalId -replace '.*/') -or $htManagementGroupsMgPath.($entry.externalId -replace '.*/').path -contains $ManagementGroupId) { + if ($htManagementGroupsMgPath.($ManagementGroupId).ParentNameChain -contains ($entryIdGuid) -or $htManagementGroupsMgPath.($entryIdGuid).path -contains $ManagementGroupId) { $null = $scopesToIterate.Add($entry) } } if ($entry.type -eq 'subscription') { - if ($htSubscriptionsMgPath.($entry.externalId -replace '.*/').ParentNameChain -contains $ManagementGroupId) { - if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) { - Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" + if ($htSubscriptionsMgPath.($entryIdGuid).ParentNameChain -contains $ManagementGroupId) { + if ($htOutOfScopeSubscriptions.($entryIdGuid)) { + Write-Host "excluding subscription $($entryIdGuid) (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entryIdGuid).outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" } else { $null = $scopesToIterate.Add($entry) @@ -4213,8 +4215,9 @@ function getPIMEligible { } else { foreach ($entry in $res) { - if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) { - Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" + $entryIdGuid = $entry.externalId -replace '.*/' + if ($htOutOfScopeSubscriptions.($entryIdGuid)) { + Write-Host "excluding subscription $($entryIdGuid) (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entryIdGuid).outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" } else { $null = $scopesToIterate.Add($entry) @@ -4233,7 +4236,7 @@ function getPIMEligible { Write-Host " Found $($entry.Count) PIM onboarded $($entry.Name)s" } - $htPIMEligibleDirect = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htPIMEligibleDirect = [System.Collections.Hashtable]::Synchronized(@{}) $relevantSubscriptionIds = $subsToProcessInCustomDataCollection.subscriptionId if ($scopesToIterate.Count -gt 0) { @@ -4403,13 +4406,12 @@ function getPIMEligible { Write-Host "Getting PIM Eligible assignments processing duration: $((New-TimeSpan -Start $start -End $end).TotalMinutes) minutes ($((New-TimeSpan -Start $start -End $end).TotalSeconds) seconds)" } function getPolicyHash { - [CmdletBinding()] param ( [Parameter(Mandatory)] [string] $json ) - return [System.BitConverter]::ToString([System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($json))) + return [string]([System.BitConverter]::ToString([System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($json)))) } function getPolicyRemediation { $currentTask = 'Getting NonCompliant (dine/modify)' @@ -5038,6 +5040,11 @@ function processALZPolicyVersionChecker { Write-Host " Switching to directory '$($ALZPath)/Enterprise-Scale'" Set-Location "$($ALZPath)/Enterprise-Scale" + #devSkim ... + $ALZCommitIdP1 = '3476914f9ba9a8f3f641a' + $ALZCommitIdP2 = '25497dfb24a4efa1017' + $ALZCommitId = "$($ALZCommitIdP1)$($ALZCommitIdP2)" + $allESLZPolicies = @{} $allESLZPolicySets = @{} $allESLZPolicyHashes = @{} @@ -5049,7 +5056,7 @@ function processALZPolicyVersionChecker { $processDataPolicies = $true foreach ($commit in $gitHist | Sort-Object -Property Date) { if ($processDataPolicies) { - if ($commit.CommitId -eq '3476914f9ba9a8f3f641a25497dfb24a4efa1017') { + if ($commit.CommitId -eq $ALZCommitId) { $processDataPolicies = $false continue } @@ -5120,7 +5127,7 @@ function processALZPolicyVersionChecker { $doNewALZPolicyReadingApproach = $false foreach ($commit in $gitHist | Sort-Object -Property Date) { - if ($commit.CommitId -eq '3476914f9ba9a8f3f641a25497dfb24a4efa1017') { + if ($commit.CommitId -eq $ALZCommitId) { $doNewALZPolicyReadingApproach = $true } #Write-Host "processing commit $($commit.CommitId) - doNewALZPolicyReadingApproach: $doNewALZPolicyReadingApproach" @@ -7045,7 +7052,7 @@ function processDefinitionInsights() { $startDefinitionInsights = Get-Date Write-Host ' Building DefinitionInsights' - $md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider + $SHA256 = New-Object -TypeName System.Security.Cryptography.SHA256CryptoServiceProvider $utf8 = New-Object -TypeName System.Text.UTF8Encoding #region definitionInsightsAzurePolicy @@ -7318,7 +7325,7 @@ function processDefinitionInsights() { } $json = $($policy.Json | ConvertTo-Json -Depth 99) - $guid = ([System.BitConverter]::ToString($md5.ComputeHash($utf8.GetBytes($policy.PolicyDefinitionId)))) -replace '-' + $guid = ([System.BitConverter]::ToString($SHA256.ComputeHash($utf8.GetBytes($policy.PolicyDefinitionId)))) -replace '-' @" @@ -7607,7 +7614,7 @@ tf.init();}} $scopeDetails = "$($policySet.ScopeId) ($($htEntities.($policySet.ScopeId).DisplayName))" } $json = $($policySet.Json | ConvertTo-Json -Depth 99) - $guid = ([System.BitConverter]::ToString($md5.ComputeHash($utf8.GetBytes($policySet.PolicyDefinitionId)))) -replace '-' + $guid = ([System.BitConverter]::ToString($SHA256.ComputeHash($utf8.GetBytes($policySet.PolicyDefinitionId)))) -replace '-' @" @@ -11359,7 +11366,8 @@ paging: {results_per_page: ['Records: ', [$spectrum]]},/*state: {types: ['local_ $allPSRuleResultsUnderThisMg = [system.collections.ArrayList]@() foreach ($mg in $grpPSRuleManagementGroups) { - if ($htManagementGroupsMgPath.($mg.name -replace '.*/').path -contains $mgchild) { + $mgNameIdHlper = $mg.name -replace '.*/' + if ($htManagementGroupsMgPath.($mgNameIdHlper).path -contains $mgchild) { $allPSRuleResultsUnderThisMg.AddRange($mg.Group) } } @@ -12987,7 +12995,8 @@ btn_reset: true, highlight_keywords: true, alternate_rows: true, auto_filter: { if (-not $NoScopeInsights) { if ($scopescnter % 50 -eq 0) { $script:scopescnter = 0 - Write-Host ' append file duration: '(Measure-Command { $script:html | Add-Content -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName).html" -Encoding utf8 -Force }).TotalSeconds 'seconds' + $addContentDurationInSeconds = (Measure-Command { $script:html | Add-Content -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName).html" -Encoding utf8 -Force }).TotalSeconds + Write-Host " append file duration: $addContentDurationInSeconds seconds" $script:html = $null } } @@ -14322,16 +14331,17 @@ function processTenantSummary() { if (($tenantPolicy.RoleDefinitionIds) -ne 'n/a') { $policyRoleDefinitionsArray = @() $policyRoleDefinitionsArray = foreach ($roleDefinitionId in $tenantPolicy.RoleDefinitionIds | Sort-Object) { - if (($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').LinkToAzAdvertizer) { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').LinkToAzAdvertizer + $roleDefinitionIdGuid = $roledefinitionId -replace '.*/' + if (($htCacheDefinitionsRole).($roleDefinitionIdGuid).LinkToAzAdvertizer) { + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).LinkToAzAdvertizer } else { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name -replace '<', '<' -replace '>', '>' + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name -replace '<', '<' -replace '>', '>' } } $policyRoleDefinitionsClearArray = @() $policyRoleDefinitionsClearArray = foreach ($roleDefinitionId in $tenantPolicy.RoleDefinitionIds | Sort-Object) { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name } $policyRoleDefinitions = $policyRoleDefinitionsArray -join "$CsvDelimiterOpposite " $policyRoleDefinitionsClear = $policyRoleDefinitionsClearArray -join "$CsvDelimiterOpposite " @@ -17161,8 +17171,9 @@ extensions: [{ name: 'sort' }] $hlp = $htPolicyAssignmentRelatedRoleAssignments.($policyAssignmentAll.PolicyAssignmentId) $relatedRoleAssignments = $hlp.relatedRoleAssignments $relatedRoleAssignmentsClear = $hlp.relatedRoleAssignmentsClear - if ($htManagedIdentityDisplayName.("$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)")) { - $hlp = $htManagedIdentityDisplayName.("$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)") + $hlperVar = "$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)" + if ($htManagedIdentityDisplayName.($hlperVar)) { + $hlp = $htManagedIdentityDisplayName.($hlperVar) $policyAssignmentMI = "$($hlp.displayname) (SPObjId: $($hlp.id))" } } @@ -24396,8 +24407,9 @@ extensions: [{ name: 'sort' }] $roleDefinitionIdsArray = [System.Collections.ArrayList]@() foreach ($roleDefinitionId in ($policy).Json.properties.policyrule.then.details.roleDefinitionIds) { - if (($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/')) { - $null = $roleDefinitionIdsArray.Add("$(($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name) ($($roleDefinitionId -replace '.*/'))") + $roleDefinitionIdGuid = $roleDefinitionId -replace '.*/' + if (($htCacheDefinitionsRole).($roleDefinitionIdGuid)) { + $null = $roleDefinitionIdsArray.Add("$(($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name) ($($roleDefinitionIdGuid))") } else { Write-Host " DiagnosticsLifeCycle: unknown RoleDefinition '$roleDefinitionId'" @@ -28515,12 +28527,11 @@ extensions: [{ name: 'sort' }] } function removeInvalidFileNameChars { param( - [Parameter(Mandatory = $true, - Position = 0, - ValueFromPipeline = $true, - ValueFromPipelineByPropertyName = $true)] - [String]$Name + [Parameter(Mandatory = $true)] + [string] + $Name ) + if ($Name -like '`[Deprecated`]:*') { $Name = $Name -replace '\[Deprecated\]\:', '[Deprecated]' } @@ -28530,6 +28541,7 @@ function removeInvalidFileNameChars { if ($Name -like '`[ASC Private Preview`]:*') { $Name = $Name -replace '\[ASC Private Preview\]\:', '[ASC Private Preview]' } + return ($Name -replace ':', '_' -replace '/', '_' -replace '\\', '_' -replace '<', '_' -replace '>', '_' -replace '\*', '_' -replace '\?', '_' -replace '\|', '_' -replace '"', '_') } function ResolveObjectIds { @@ -29736,7 +29748,7 @@ function validateLeastPrivilegeForUser { $currentTask = "Get RBAC Role definition '$nonReaderRoleAssigned'" $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($nonReaderRoleAssigned)?api-version=2022-04-01" $method = 'GET' - $getRole = AzAPICall -AzAPICallConfiguration $azapicallConf -uri $uri -listenOn Content + $getRole = AzAPICall -AzAPICallConfiguration $azapicallConf -uri $uri -method $method -listenOn Content if ($getRole.properties.roleName -eq 'owner' -or $getRole.properties.roleName -eq 'contributor') { Write-Host " - $($getRole.properties.roleName) ($($getRole.properties.type)) !!!" @@ -33502,8 +33514,9 @@ function dataCollectionRoleAssignmentsMG { $pimSlotEnd = '' } - if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentId -replace '.*/')) { - $script:htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentId -replace '.*/') = @{ + $roleAssignmentIdGuid = $roleAssignmentId -replace '.*/' + if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid)) { + $script:htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid) = @{ assignment = $L0mgmtGroupRoleAssignment } } @@ -33735,11 +33748,12 @@ function dataCollectionRoleAssignmentsSub { foreach ($roleAssignmentFromAPI in $roleAssignmentsFromAPI) { if ($roleAssignmentFromAPI.id -match "/subscriptions/$($scopeId)/") { - if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentFromAPI.id -replace '.*/')) { + $roleAssignmentIdGuid = $roleAssignmentFromAPI.id -replace '.*/' + if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid)) { $null = $baseRoleAssignments.Add($roleAssignmentFromAPI) } else { - $null = $baseRoleAssignments.Add($htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentFromAPI.id -replace '.*/').assignment) + $null = $baseRoleAssignments.Add($htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid).assignment) } } else { @@ -34626,22 +34640,22 @@ $htSubDetails = @{} if (-not $HierarchyMapOnly) { #helper ht / collect results /save some time - $htCacheDefinitionsPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsPolicySet = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsRole = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsBlueprint = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htRoleAssignmentsPIM = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htCacheDefinitionsPolicy = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsPolicySet = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsRole = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsBlueprint = [System.Collections.Hashtable]::Synchronized(@{}) + $htRoleAssignmentsPIM = [System.Collections.Hashtable]::Synchronized(@{}) $htPoliciesUsedInPolicySets = @{} - $htSubscriptionTags = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsPolicyOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsRole = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsRBACOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsBlueprint = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceMG = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceSUB = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceResponseTooLargeMG = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceResponseTooLargeSUB = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionTags = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsPolicyOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsRole = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsRBACOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsBlueprint = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsPolicy = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceMG = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceSUB = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceResponseTooLargeMG = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceResponseTooLargeSUB = [System.Collections.Hashtable]::Synchronized(@{}) $outOfScopeSubscriptions = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htOutOfScopeSubscriptions = @{} if ($azAPICallConf['htParameters'].DoAzureConsumption -eq $true) { @@ -34657,22 +34671,22 @@ if (-not $HierarchyMapOnly) { } } $customDataCollectionDuration = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceLocks = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.AllScopes = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.Subscription = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.ResourceGroup = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.Resource = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceLocks = [System.Collections.Hashtable]::Synchronized(@{}) + $htAllTagList = [System.Collections.Hashtable]::Synchronized(@{}) + $htAllTagList.AllScopes = @{} + $htAllTagList.Subscription = @{} + $htAllTagList.ResourceGroup = @{} + $htAllTagList.Resource = @{} $arrayTagList = [System.Collections.ArrayList]@() - $htSubscriptionTagList = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htPolicyAssignmentExemptions = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htUserTypesGuest = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionTagList = [System.Collections.Hashtable]::Synchronized(@{}) + $htPolicyAssignmentExemptions = [System.Collections.Hashtable]::Synchronized(@{}) + $htUserTypesGuest = [System.Collections.Hashtable]::Synchronized(@{}) $resourcesAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $resourcesIdsAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $resourceGroupsAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceProvidersAll = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceProvidersAll = [System.Collections.Hashtable]::Synchronized(@{}) $arrayFeaturesAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceTypesUniqueResource = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceTypesUniqueResource = [System.Collections.Hashtable]::Synchronized(@{}) $arrayDataCollectionProgressMg = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayDataCollectionProgressSub = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arraySubResourcesAddArrayDuration = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -34680,38 +34694,38 @@ if (-not $HierarchyMapOnly) { $htDiagnosticSettingsMgSub = @{} $htDiagnosticSettingsMgSub.mg = @{} $htDiagnosticSettingsMgSub.sub = @{} - $htMgAtScopePolicyAssignments = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htMgAtScopePoliciesScoped = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htMgAtScopeRoleAssignments = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htMgAtScopePolicyAssignments = [System.Collections.Hashtable]::Synchronized(@{}) + $htMgAtScopePoliciesScoped = [System.Collections.Hashtable]::Synchronized(@{}) + $htMgAtScopeRoleAssignments = [System.Collections.Hashtable]::Synchronized(@{}) $htMgASCSecureScore = @{} - $htConsumptionExceptionLog = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htConsumptionExceptionLog.Mg = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htConsumptionExceptionLog.Sub = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htRoleAssignmentsFromAPIInheritancePrevention = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.PolicyAssignment = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Policy = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.PolicySet = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Role = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Subscription = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.ManagementGroup = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htPrincipals = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htServicePrincipals = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htConsumptionExceptionLog = [System.Collections.Hashtable]::Synchronized(@{}) + $htConsumptionExceptionLog.Mg = @{} + $htConsumptionExceptionLog.Sub = @{} + $htRoleAssignmentsFromAPIInheritancePrevention = [System.Collections.Hashtable]::Synchronized(@{}) + $htNamingValidation = [System.Collections.Hashtable]::Synchronized(@{}) + $htNamingValidation.PolicyAssignment = @{} + $htNamingValidation.Policy = @{} + $htNamingValidation.PolicySet = @{} + $htNamingValidation.Role = @{} + $htNamingValidation.Subscription = @{} + $htNamingValidation.ManagementGroup = @{} + $htPrincipals = [System.Collections.Hashtable]::Synchronized(@{}) + $htServicePrincipals = [System.Collections.Hashtable]::Synchronized(@{}) $htDailySummary = @{} $arrayDefenderPlans = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayDefenderPlansSubscriptionsSkipped = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayUserAssignedIdentities4Resources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htSubscriptionsRoleAssignmentLimit = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionsRoleAssignmentLimit = [System.Collections.Hashtable]::Synchronized(@{}) if ($azAPICallConf['htParameters'].NoMDfCSecureScore -eq $false) { $htMgASCSecureScore = @{} } $htManagedIdentityForPolicyAssignment = @{} $htPolicyAssignmentManagedIdentity = @{} $htManagedIdentityDisplayName = @{} - $htAppDetails = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAppDetails = [System.Collections.Hashtable]::Synchronized(@{}) if (-not $NoAADGroupsResolveMembers) { - $htAADGroupsDetails = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAADGroupsExeedingMemberLimit = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAADGroupsDetails = [System.Collections.Hashtable]::Synchronized(@{}) + $htAADGroupsExeedingMemberLimit = [System.Collections.Hashtable]::Synchronized(@{}) $arrayGroupRoleAssignmentsOnServicePrincipals = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayGroupRequestResourceNotFound = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayProgressedAADGroups = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -34721,28 +34735,27 @@ if (-not $HierarchyMapOnly) { } $arrayPsRule = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPSRuleTracking = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized(@{}) $arrayOrphanedResources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPIMEligible = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $alzPolicies = @{} $alzPolicySets = @{} $alzPolicyHashes = @{} $alzPolicySetHashes = @{} - $htDoARMRoleAssignmentScheduleInstances = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htDoARMRoleAssignmentScheduleInstances = [System.Collections.Hashtable]::Synchronized(@{}) $htDoARMRoleAssignmentScheduleInstances.Do = $true $storageAccounts = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayStorageAccountAnalysisResults = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htDefenderEmailContacts = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) + $htDefenderEmailContacts = [System.Collections.Hashtable]::Synchronized(@{}) $arrayVNets = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPrivateEndPoints = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPrivateEndPointsFromResourceProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htUnknownTenantsForSubscription = @{} - $htResourcePropertiesConvertfromJSONFailed = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - #$htResourcesWithProperties = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourcePropertiesConvertfromJSONFailed = [System.Collections.Hashtable]::Synchronized(@{}) $htResourceProvidersRef = @{} - $htAvailablePrivateEndpointTypes = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAvailablePrivateEndpointTypes = [System.Collections.Hashtable]::Synchronized(@{}) $arrayAdvisorScores = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htHashesBuiltInPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htHashesBuiltInPolicy = [System.Collections.Hashtable]::Synchronized(@{}) $arrayCustomBuiltInPolicyParity = [System.Collections.ArrayList]@() $arrayRemediatable = [System.Collections.ArrayList]@() } @@ -35065,15 +35078,16 @@ if (-not $HierarchyMapOnly) { if (-not [string]::IsNullOrWhiteSpace($htCacheDefinitionsPolicy.($policyDefinitionId).Json.properties.policyRule.then.details.roleDefinitionIds)) { foreach ($roledefinitionId in $htCacheDefinitionsPolicy.($policyDefinitionId).Json.properties.policyRule.then.details.roleDefinitionIds) { if (-not [string]::IsNullOrWhitespace($roledefinitionId)) { - if (-not $htCacheDefinitionsRole.($roledefinitionId -replace '.*/')) { + $roleDefinitionIdGuid = $roledefinitionId -replace '.*/' + if (-not $htCacheDefinitionsRole.($roleDefinitionIdGuid)) { Write-Host "Finding: policyDefinitionId '$($policyDefinitionId)' has unknown roleDefinitionId '$roledefinitionId' in policyRule.then.details.roleDefinitionIds" -ForegroundColor DarkRed } else { - if (-not $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/')) { - $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/') = [System.Collections.ArrayList]@() + if (-not $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid)) { + $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid) = [System.Collections.ArrayList]@() } try { - $null = $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/').Add($policyDefinitionId) + $null = $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid).Add($policyDefinitionId) } catch { Write-Host "policyDefinitionId '$($policyDefinitionId)' JSON:" diff --git a/pwsh/dev/buildAzGovVizParallel.ps1 b/pwsh/dev/buildAzGovVizParallel.ps1 index 73e75628..0d5e2ac5 100644 --- a/pwsh/dev/buildAzGovVizParallel.ps1 +++ b/pwsh/dev/buildAzGovVizParallel.ps1 @@ -1,4 +1,4 @@ -param( +param( [switch] $skipVersionCompare ) @@ -20,7 +20,7 @@ $endIndex = $AzGovVizScriptFile.IndexOf('#endregion Functions') $textBefore = $AzGovVizScriptFile.SubString(0, $startIndex) $textAfter = $AzGovVizScriptFile.SubString($endIndex) -$textBefore.TrimEnd(), $newContent, $textAfter | Set-Content -Path .\pwsh\AzGovVizParallel.ps1 +$textBefore.TrimEnd(), $newContent, $textAfter | Set-Content -Path .\pwsh\AzGovVizParallel.ps1 -Encoding utf8BOM $versionPattern = 'ProductVersion = ' $versiontxt = (Select-String -Path .\pwsh\AzGovVizParallel.ps1 -Pattern $versionPattern) -replace ".*$versionPattern" -replace "'" -replace ',' diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 93e56b22..d9f5c8e2 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -1,4 +1,4 @@ -<# +<# .SYNOPSIS This script creates the following files to help better understand and audit your governance setup csv file @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.5', + $ProductVersion = '6.4.6', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -403,7 +403,7 @@ Param $NoCsvExport, [string] - [parameter(ValueFromPipeline)][ValidateSet(';', ',')][string]$CsvDelimiter = ';', + [ValidateSet(';', ',')]$CsvDelimiter = ';', [switch] $CsvExportUseQuotesAsNeeded, @@ -483,7 +483,7 @@ Param $DoNotIncludeResourceGroupsAndResourcesOnRBAC, [Alias('AzureDevOpsWikiHierarchyDirection')] - [parameter(ValueFromPipeline)][ValidateSet('TD', 'LR')][string]$MermaidDirection = 'TD', + [ValidateSet('TD', 'LR')][string]$MermaidDirection = 'TD', [int] $ChangeTrackingDays = 14, @@ -639,7 +639,7 @@ Set-Item Env:\SuppressAzurePowerShellBreakingChangeWarnings 'true' #start $startAzGovViz = Get-Date $startTime = Get-Date -Format 'dd-MMM-yyyy HH:mm:ss' -Write-Host "Start Azure Governance Visualizer $($startTime) (#$($ProductVersion))" +Write-Host "Start Azure Governance Visualizer (aka $Product) $($startTime) (#$($ProductVersion))" if ($ManagementGroupId -match ' ') { Write-Host "Provided Management Group ID: '$($ManagementGroupId)'" -ForegroundColor Yellow @@ -916,22 +916,22 @@ $htSubDetails = @{} if (-not $HierarchyMapOnly) { #helper ht / collect results /save some time - $htCacheDefinitionsPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsPolicySet = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsRole = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheDefinitionsBlueprint = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htRoleAssignmentsPIM = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htCacheDefinitionsPolicy = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsPolicySet = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsRole = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheDefinitionsBlueprint = [System.Collections.Hashtable]::Synchronized(@{}) + $htRoleAssignmentsPIM = [System.Collections.Hashtable]::Synchronized(@{}) $htPoliciesUsedInPolicySets = @{} - $htSubscriptionTags = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsPolicyOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsRole = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsRBACOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsBlueprint = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCacheAssignmentsPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceMG = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceSUB = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceResponseTooLargeMG = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htCachePolicyComplianceResponseTooLargeSUB = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionTags = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsPolicyOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsRole = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsRBACOnResourceGroupsAndResources = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsBlueprint = [System.Collections.Hashtable]::Synchronized(@{}) + $htCacheAssignmentsPolicy = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceMG = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceSUB = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceResponseTooLargeMG = [System.Collections.Hashtable]::Synchronized(@{}) + $htCachePolicyComplianceResponseTooLargeSUB = [System.Collections.Hashtable]::Synchronized(@{}) $outOfScopeSubscriptions = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htOutOfScopeSubscriptions = @{} if ($azAPICallConf['htParameters'].DoAzureConsumption -eq $true) { @@ -947,22 +947,22 @@ if (-not $HierarchyMapOnly) { } } $customDataCollectionDuration = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceLocks = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.AllScopes = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.Subscription = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.ResourceGroup = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAllTagList.Resource = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceLocks = [System.Collections.Hashtable]::Synchronized(@{}) + $htAllTagList = [System.Collections.Hashtable]::Synchronized(@{}) + $htAllTagList.AllScopes = @{} + $htAllTagList.Subscription = @{} + $htAllTagList.ResourceGroup = @{} + $htAllTagList.Resource = @{} $arrayTagList = [System.Collections.ArrayList]@() - $htSubscriptionTagList = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htPolicyAssignmentExemptions = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htUserTypesGuest = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionTagList = [System.Collections.Hashtable]::Synchronized(@{}) + $htPolicyAssignmentExemptions = [System.Collections.Hashtable]::Synchronized(@{}) + $htUserTypesGuest = [System.Collections.Hashtable]::Synchronized(@{}) $resourcesAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $resourcesIdsAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $resourceGroupsAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceProvidersAll = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceProvidersAll = [System.Collections.Hashtable]::Synchronized(@{}) $arrayFeaturesAll = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htResourceTypesUniqueResource = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourceTypesUniqueResource = [System.Collections.Hashtable]::Synchronized(@{}) $arrayDataCollectionProgressMg = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayDataCollectionProgressSub = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arraySubResourcesAddArrayDuration = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -970,38 +970,38 @@ if (-not $HierarchyMapOnly) { $htDiagnosticSettingsMgSub = @{} $htDiagnosticSettingsMgSub.mg = @{} $htDiagnosticSettingsMgSub.sub = @{} - $htMgAtScopePolicyAssignments = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htMgAtScopePoliciesScoped = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htMgAtScopeRoleAssignments = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htMgAtScopePolicyAssignments = [System.Collections.Hashtable]::Synchronized(@{}) + $htMgAtScopePoliciesScoped = [System.Collections.Hashtable]::Synchronized(@{}) + $htMgAtScopeRoleAssignments = [System.Collections.Hashtable]::Synchronized(@{}) $htMgASCSecureScore = @{} - $htConsumptionExceptionLog = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htConsumptionExceptionLog.Mg = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htConsumptionExceptionLog.Sub = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htRoleAssignmentsFromAPIInheritancePrevention = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.PolicyAssignment = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Policy = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.PolicySet = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Role = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.Subscription = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htNamingValidation.ManagementGroup = @{} #[System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htPrincipals = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htServicePrincipals = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htConsumptionExceptionLog = [System.Collections.Hashtable]::Synchronized(@{}) + $htConsumptionExceptionLog.Mg = @{} + $htConsumptionExceptionLog.Sub = @{} + $htRoleAssignmentsFromAPIInheritancePrevention = [System.Collections.Hashtable]::Synchronized(@{}) + $htNamingValidation = [System.Collections.Hashtable]::Synchronized(@{}) + $htNamingValidation.PolicyAssignment = @{} + $htNamingValidation.Policy = @{} + $htNamingValidation.PolicySet = @{} + $htNamingValidation.Role = @{} + $htNamingValidation.Subscription = @{} + $htNamingValidation.ManagementGroup = @{} + $htPrincipals = [System.Collections.Hashtable]::Synchronized(@{}) + $htServicePrincipals = [System.Collections.Hashtable]::Synchronized(@{}) $htDailySummary = @{} $arrayDefenderPlans = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayDefenderPlansSubscriptionsSkipped = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayUserAssignedIdentities4Resources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htSubscriptionsRoleAssignmentLimit = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htSubscriptionsRoleAssignmentLimit = [System.Collections.Hashtable]::Synchronized(@{}) if ($azAPICallConf['htParameters'].NoMDfCSecureScore -eq $false) { $htMgASCSecureScore = @{} } $htManagedIdentityForPolicyAssignment = @{} $htPolicyAssignmentManagedIdentity = @{} $htManagedIdentityDisplayName = @{} - $htAppDetails = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAppDetails = [System.Collections.Hashtable]::Synchronized(@{}) if (-not $NoAADGroupsResolveMembers) { - $htAADGroupsDetails = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - $htAADGroupsExeedingMemberLimit = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAADGroupsDetails = [System.Collections.Hashtable]::Synchronized(@{}) + $htAADGroupsExeedingMemberLimit = [System.Collections.Hashtable]::Synchronized(@{}) $arrayGroupRoleAssignmentsOnServicePrincipals = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayGroupRequestResourceNotFound = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayProgressedAADGroups = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -1011,28 +1011,27 @@ if (-not $HierarchyMapOnly) { } $arrayPsRule = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPSRuleTracking = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htClassicAdministrators = [System.Collections.Hashtable]::Synchronized(@{}) $arrayOrphanedResources = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPIMEligible = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $alzPolicies = @{} $alzPolicySets = @{} $alzPolicyHashes = @{} $alzPolicySetHashes = @{} - $htDoARMRoleAssignmentScheduleInstances = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htDoARMRoleAssignmentScheduleInstances = [System.Collections.Hashtable]::Synchronized(@{}) $htDoARMRoleAssignmentScheduleInstances.Do = $true $storageAccounts = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayStorageAccountAnalysisResults = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htDefenderEmailContacts = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) + $htDefenderEmailContacts = [System.Collections.Hashtable]::Synchronized(@{}) $arrayVNets = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPrivateEndPoints = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $arrayPrivateEndPointsFromResourceProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) $htUnknownTenantsForSubscription = @{} - $htResourcePropertiesConvertfromJSONFailed = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} - #$htResourcesWithProperties = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htResourcePropertiesConvertfromJSONFailed = [System.Collections.Hashtable]::Synchronized(@{}) $htResourceProvidersRef = @{} - $htAvailablePrivateEndpointTypes = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htAvailablePrivateEndpointTypes = [System.Collections.Hashtable]::Synchronized(@{}) $arrayAdvisorScores = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) - $htHashesBuiltInPolicy = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htHashesBuiltInPolicy = [System.Collections.Hashtable]::Synchronized(@{}) $arrayCustomBuiltInPolicyParity = [System.Collections.ArrayList]@() $arrayRemediatable = [System.Collections.ArrayList]@() } @@ -1355,15 +1354,16 @@ if (-not $HierarchyMapOnly) { if (-not [string]::IsNullOrWhiteSpace($htCacheDefinitionsPolicy.($policyDefinitionId).Json.properties.policyRule.then.details.roleDefinitionIds)) { foreach ($roledefinitionId in $htCacheDefinitionsPolicy.($policyDefinitionId).Json.properties.policyRule.then.details.roleDefinitionIds) { if (-not [string]::IsNullOrWhitespace($roledefinitionId)) { - if (-not $htCacheDefinitionsRole.($roledefinitionId -replace '.*/')) { + $roleDefinitionIdGuid = $roledefinitionId -replace '.*/' + if (-not $htCacheDefinitionsRole.($roleDefinitionIdGuid)) { Write-Host "Finding: policyDefinitionId '$($policyDefinitionId)' has unknown roleDefinitionId '$roledefinitionId' in policyRule.then.details.roleDefinitionIds" -ForegroundColor DarkRed } else { - if (-not $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/')) { - $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/') = [System.Collections.ArrayList]@() + if (-not $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid)) { + $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid) = [System.Collections.ArrayList]@() } try { - $null = $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId -replace '.*/').Add($policyDefinitionId) + $null = $htRoleDefinitionIdsUsedInPolicy.($roleDefinitionIdGuid).Add($policyDefinitionId) } catch { Write-Host "policyDefinitionId '$($policyDefinitionId)' JSON:" diff --git a/pwsh/dev/functions/addHtParameters.ps1 b/pwsh/dev/functions/addHtParameters.ps1 index 964f7168..9e6d413a 100644 --- a/pwsh/dev/functions/addHtParameters.ps1 +++ b/pwsh/dev/functions/addHtParameters.ps1 @@ -1,4 +1,4 @@ -function addHtParameters { +function addHtParameters { Write-Host 'Add Azure Governance Visualizer htParameters' if ($LargeTenant -eq $true) { $script:NoScopeInsights = $true diff --git a/pwsh/dev/functions/addIndexNumberToArray.ps1 b/pwsh/dev/functions/addIndexNumberToArray.ps1 index 49505713..1d46937e 100644 --- a/pwsh/dev/functions/addIndexNumberToArray.ps1 +++ b/pwsh/dev/functions/addIndexNumberToArray.ps1 @@ -1,4 +1,4 @@ -function addIndexNumberToArray ( +function addIndexNumberToArray ( [Parameter(Mandatory = $True)] [array]$array ) { diff --git a/pwsh/dev/functions/addRowToTable.ps1 b/pwsh/dev/functions/addRowToTable.ps1 index d0717772..0f91454b 100644 --- a/pwsh/dev/functions/addRowToTable.ps1 +++ b/pwsh/dev/functions/addRowToTable.ps1 @@ -1,4 +1,4 @@ -function addRowToTable() { +function addRowToTable() { Param ( [string]$level = 0, [string]$mgName = '', diff --git a/pwsh/dev/functions/apiCallTracking.ps1 b/pwsh/dev/functions/apiCallTracking.ps1 index b9129382..fd46a1f3 100644 --- a/pwsh/dev/functions/apiCallTracking.ps1 +++ b/pwsh/dev/functions/apiCallTracking.ps1 @@ -1,4 +1,4 @@ -function apiCallTracking { +function apiCallTracking { [CmdletBinding()]Param( [string]$stage, [string]$spacing diff --git a/pwsh/dev/functions/buildJSON.ps1 b/pwsh/dev/functions/buildJSON.ps1 index 71b7e65f..55b18469 100644 --- a/pwsh/dev/functions/buildJSON.ps1 +++ b/pwsh/dev/functions/buildJSON.ps1 @@ -1,4 +1,4 @@ -function buildJSON { +function buildJSON { #$fileTimestamp = Get-Date -Format "yyyyMM-dd HHmmss" $startJSON = Get-Date $startBuildHt = Get-Date diff --git a/pwsh/dev/functions/buildMD.ps1 b/pwsh/dev/functions/buildMD.ps1 index cef442b3..812556f6 100644 --- a/pwsh/dev/functions/buildMD.ps1 +++ b/pwsh/dev/functions/buildMD.ps1 @@ -1,4 +1,4 @@ -function buildMD { +function buildMD { Write-Host 'Building Markdown' $startBuildMD = Get-Date $script:arrayMgs = [System.Collections.ArrayList]@() diff --git a/pwsh/dev/functions/buildPolicyAllJSON.ps1 b/pwsh/dev/functions/buildPolicyAllJSON.ps1 index 6d894a09..0a3f1a46 100644 --- a/pwsh/dev/functions/buildPolicyAllJSON.ps1 +++ b/pwsh/dev/functions/buildPolicyAllJSON.ps1 @@ -1,4 +1,4 @@ -function buildPolicyAllJSON { +function buildPolicyAllJSON { Write-Host 'Creating PolicyAll JSON' $startPolicyAllJSON = Get-Date $htPolicyAndPolicySet = [ordered]@{} diff --git a/pwsh/dev/functions/buildTree.ps1 b/pwsh/dev/functions/buildTree.ps1 index d75fc5d9..5bd07f99 100644 --- a/pwsh/dev/functions/buildTree.ps1 +++ b/pwsh/dev/functions/buildTree.ps1 @@ -1,4 +1,4 @@ -function buildTree($mgId, $prnt) { +function buildTree($mgId, $prnt) { $getMg = $htEntities.values.where( { $_.type -eq 'Microsoft.Management/managementGroups' -and $_.id -eq $mgId }) $childrenManagementGroups = $htEntities.values.where( { $_.type -eq 'Microsoft.Management/managementGroups' -and $_.parentId -eq "/providers/Microsoft.Management/managementGroups/$($getMg.Id)" }) $mgNameValid = removeInvalidFileNameChars $getMg.Id diff --git a/pwsh/dev/functions/cacheBuiltIn.ps1 b/pwsh/dev/functions/cacheBuiltIn.ps1 index dc9d0ea8..0c82101f 100644 --- a/pwsh/dev/functions/cacheBuiltIn.ps1 +++ b/pwsh/dev/functions/cacheBuiltIn.ps1 @@ -1,4 +1,4 @@ -function cacheBuiltIn { +function cacheBuiltIn { $startDefinitionsCaching = Get-Date Write-Host 'Caching built-in Policy and RBAC Role definitions' diff --git a/pwsh/dev/functions/checkAzGovVizVersion.ps1 b/pwsh/dev/functions/checkAzGovVizVersion.ps1 index 6a083715..ac025cfb 100644 --- a/pwsh/dev/functions/checkAzGovVizVersion.ps1 +++ b/pwsh/dev/functions/checkAzGovVizVersion.ps1 @@ -1,4 +1,4 @@ -function checkAzGovVizVersion { +function checkAzGovVizVersion { try { $getRepoVersion = Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/JulianHayward/Azure-MG-Sub-Governance-Reporting/master/version.json' $repoVersion = ($getRepoVersion.Content | ConvertFrom-Json).ProductVersion diff --git a/pwsh/dev/functions/createTagList.ps1 b/pwsh/dev/functions/createTagList.ps1 index 3296018a..73393512 100644 --- a/pwsh/dev/functions/createTagList.ps1 +++ b/pwsh/dev/functions/createTagList.ps1 @@ -1,4 +1,4 @@ -function createTagList { +function createTagList { $startTagListArray = Get-Date Write-Host 'Creating TagList array' diff --git a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 index 014a9c0f..60f341b5 100644 --- a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 +++ b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 @@ -1,4 +1,4 @@ -#region functions4DataCollection +#region functions4DataCollection function dataCollectionMGSecureScore { [CmdletBinding()]Param( @@ -3615,8 +3615,9 @@ function dataCollectionRoleAssignmentsMG { $pimSlotEnd = '' } - if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentId -replace '.*/')) { - $script:htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentId -replace '.*/') = @{ + $roleAssignmentIdGuid = $roleAssignmentId -replace '.*/' + if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid)) { + $script:htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid) = @{ assignment = $L0mgmtGroupRoleAssignment } } @@ -3848,11 +3849,12 @@ function dataCollectionRoleAssignmentsSub { foreach ($roleAssignmentFromAPI in $roleAssignmentsFromAPI) { if ($roleAssignmentFromAPI.id -match "/subscriptions/$($scopeId)/") { - if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentFromAPI.id -replace '.*/')) { + $roleAssignmentIdGuid = $roleAssignmentFromAPI.id -replace '.*/' + if (-not $htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid)) { $null = $baseRoleAssignments.Add($roleAssignmentFromAPI) } else { - $null = $baseRoleAssignments.Add($htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentFromAPI.id -replace '.*/').assignment) + $null = $baseRoleAssignments.Add($htRoleAssignmentsFromAPIInheritancePrevention.($roleAssignmentIdGuid).assignment) } } else { diff --git a/pwsh/dev/functions/detailSubscriptions.ps1 b/pwsh/dev/functions/detailSubscriptions.ps1 index 5bc78137..09ae7197 100644 --- a/pwsh/dev/functions/detailSubscriptions.ps1 +++ b/pwsh/dev/functions/detailSubscriptions.ps1 @@ -1,4 +1,4 @@ -function detailSubscriptions { +function detailSubscriptions { $start = Get-Date Write-Host 'Subscription picking' #API in rare cases returns duplicates, therefor sorting unique (id) diff --git a/pwsh/dev/functions/detectPolicyEffect.ps1 b/pwsh/dev/functions/detectPolicyEffect.ps1 index 14fa0ad5..b1cb14d1 100644 --- a/pwsh/dev/functions/detectPolicyEffect.ps1 +++ b/pwsh/dev/functions/detectPolicyEffect.ps1 @@ -1,4 +1,4 @@ -function detectPolicyEffect { +function detectPolicyEffect { [CmdletBinding()] Param ( diff --git a/pwsh/dev/functions/exportBaseCSV.ps1 b/pwsh/dev/functions/exportBaseCSV.ps1 index 6b9b593e..6d0a4c38 100644 --- a/pwsh/dev/functions/exportBaseCSV.ps1 +++ b/pwsh/dev/functions/exportBaseCSV.ps1 @@ -1,4 +1,4 @@ -function exportBaseCSV { +function exportBaseCSV { if (-not $NoCsvExport) { Write-Host "Exporting CSV '$($outputPath)$($DirectorySeparatorChar)$($fileName).csv'" $startBuildCSV = Get-Date diff --git a/pwsh/dev/functions/exportResourceLocks.ps1 b/pwsh/dev/functions/exportResourceLocks.ps1 index cbf9a12c..9b0045d4 100644 --- a/pwsh/dev/functions/exportResourceLocks.ps1 +++ b/pwsh/dev/functions/exportResourceLocks.ps1 @@ -1,4 +1,4 @@ -function exportResourceLocks { +function exportResourceLocks { $arrayResourceLocks4CSV = [System.Collections.ArrayList]@() foreach ($sub in $htResourceLocks.Keys) { $hlper = $htSubscriptionsMgPath.($sub) diff --git a/pwsh/dev/functions/getConsumption.ps1 b/pwsh/dev/functions/getConsumption.ps1 index fa00eca1..885962fd 100644 --- a/pwsh/dev/functions/getConsumption.ps1 +++ b/pwsh/dev/functions/getConsumption.ps1 @@ -1,4 +1,4 @@ -function getConsumption { +function getConsumption { function addToAllConsumptionData { [CmdletBinding()]Param( diff --git a/pwsh/dev/functions/getDefaultManagementGroup.ps1 b/pwsh/dev/functions/getDefaultManagementGroup.ps1 index 61193635..804a91b9 100644 --- a/pwsh/dev/functions/getDefaultManagementGroup.ps1 +++ b/pwsh/dev/functions/getDefaultManagementGroup.ps1 @@ -1,4 +1,4 @@ -function getDefaultManagementGroup { +function getDefaultManagementGroup { $currentTask = 'Get Default Management Group' Write-Host $currentTask #https://learn.microsoft.com/azure/governance/management-groups/how-to/protect-resource-hierarchy#setting---default-management-group diff --git a/pwsh/dev/functions/getEntities.ps1 b/pwsh/dev/functions/getEntities.ps1 index db018623..b24f4619 100644 --- a/pwsh/dev/functions/getEntities.ps1 +++ b/pwsh/dev/functions/getEntities.ps1 @@ -1,4 +1,4 @@ -function getEntities { +function getEntities { Write-Host 'Entities' $startEntities = Get-Date $currentTask = ' Getting Entities' @@ -59,11 +59,12 @@ function getEntities { $array = $entity.properties.parentNameChain $array += $entity.name + $script:htSubscriptionsMgPath.($entity.name) = @{ ParentNameChain = $entity.properties.parentNameChain ParentNameChainDelimited = $entity.properties.parentNameChain -join '/' - Parent = $entity.properties.parent.Id -replace '.*/' - ParentName = $htEntitiesPlain.($entity.properties.parent.Id -replace '.*/').properties.displayName + Parent = $parent + ParentName = $htEntitiesPlain.($parent).properties.displayName DisplayName = $entity.properties.displayName path = $array pathDelimited = $array -join '/' diff --git a/pwsh/dev/functions/getFileNaming.ps1 b/pwsh/dev/functions/getFileNaming.ps1 index 4513abc3..86f9d66e 100644 --- a/pwsh/dev/functions/getFileNaming.ps1 +++ b/pwsh/dev/functions/getFileNaming.ps1 @@ -1,4 +1,4 @@ -function getFileNaming { +function getFileNaming { if ($azAPICallConf['htParameters'].onAzureDevOpsOrGitHubActions -eq $true) { if ($HierarchyMapOnly) { $script:fileName = "AzGovViz_HierarchyMapOnly_$($ManagementGroupId)" diff --git a/pwsh/dev/functions/getGroupmembers.ps1 b/pwsh/dev/functions/getGroupmembers.ps1 index 7f52e1a4..65fd7f4a 100644 --- a/pwsh/dev/functions/getGroupmembers.ps1 +++ b/pwsh/dev/functions/getGroupmembers.ps1 @@ -1,4 +1,4 @@ - + function getGroupmembers($aadGroupId, $aadGroupDisplayName) { if (-not $htAADGroupsDetails.($aadGroupId)) { $script:htAADGroupsDetails.$aadGroupId = @{ diff --git a/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 b/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 index f86d14cb..4c59b538 100644 --- a/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 +++ b/pwsh/dev/functions/getMDfCSecureScoreMG.ps1 @@ -1,4 +1,4 @@ -function getMDfCSecureScoreMG { +function getMDfCSecureScoreMG { $start = Get-Date $currentTask = 'Getting Microsoft Defender for Cloud Secure Score for Management Groups' Write-Host $currentTask diff --git a/pwsh/dev/functions/getOrphanedResources.ps1 b/pwsh/dev/functions/getOrphanedResources.ps1 index 2fbf5807..4bdee614 100644 --- a/pwsh/dev/functions/getOrphanedResources.ps1 +++ b/pwsh/dev/functions/getOrphanedResources.ps1 @@ -1,4 +1,4 @@ -function getOrphanedResources { +function getOrphanedResources { $start = Get-Date Write-Host 'Getting orphaned/unused resources (ARG)' diff --git a/pwsh/dev/functions/getPIMEligible.ps1 b/pwsh/dev/functions/getPIMEligible.ps1 index d97185bb..ea66f9fc 100644 --- a/pwsh/dev/functions/getPIMEligible.ps1 +++ b/pwsh/dev/functions/getPIMEligible.ps1 @@ -1,4 +1,4 @@ -function getPIMEligible { +function getPIMEligible { $start = Get-Date $currentTask = 'Get PIM onboarded Subscriptions and Management Groups' @@ -12,15 +12,16 @@ function getPIMEligible { if (-not $PIMEligibilityIgnoreScope) { if (($azAPICallConf['checkContext']).Tenant.Id -ne $ManagementGroupId) { foreach ($entry in $res) { + $entryIdGuid = $entry.externalId -replace '.*/' if ($entry.type -eq 'managementGroup') { - if ($htManagementGroupsMgPath.($ManagementGroupId).ParentNameChain -contains ($entry.externalId -replace '.*/') -or $htManagementGroupsMgPath.($entry.externalId -replace '.*/').path -contains $ManagementGroupId) { + if ($htManagementGroupsMgPath.($ManagementGroupId).ParentNameChain -contains ($entryIdGuid) -or $htManagementGroupsMgPath.($entryIdGuid).path -contains $ManagementGroupId) { $null = $scopesToIterate.Add($entry) } } if ($entry.type -eq 'subscription') { - if ($htSubscriptionsMgPath.($entry.externalId -replace '.*/').ParentNameChain -contains $ManagementGroupId) { - if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) { - Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" + if ($htSubscriptionsMgPath.($entryIdGuid).ParentNameChain -contains $ManagementGroupId) { + if ($htOutOfScopeSubscriptions.($entryIdGuid)) { + Write-Host "excluding subscription $($entryIdGuid) (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entryIdGuid).outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" } else { $null = $scopesToIterate.Add($entry) @@ -31,8 +32,9 @@ function getPIMEligible { } else { foreach ($entry in $res) { - if ($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/')) { - Write-Host "excluding subscription $($entry.externalId -replace '.*/') (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entry.externalId -replace '.*/').outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" + $entryIdGuid = $entry.externalId -replace '.*/' + if ($htOutOfScopeSubscriptions.($entryIdGuid)) { + Write-Host "excluding subscription $($entryIdGuid) (outOfScopeSubscription -> $($htOutOfScopeSubscriptions.($entryIdGuid).outOfScopeReason)) (`$PIMEligibilityIgnoreScope=$PIMEligibilityIgnoreScope)" } else { $null = $scopesToIterate.Add($entry) @@ -51,7 +53,7 @@ function getPIMEligible { Write-Host " Found $($entry.Count) PIM onboarded $($entry.Name)s" } - $htPIMEligibleDirect = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable)) #@{} + $htPIMEligibleDirect = [System.Collections.Hashtable]::Synchronized(@{}) $relevantSubscriptionIds = $subsToProcessInCustomDataCollection.subscriptionId if ($scopesToIterate.Count -gt 0) { diff --git a/pwsh/dev/functions/getPolicyHash.ps1 b/pwsh/dev/functions/getPolicyHash.ps1 index 277905f8..180ea6c2 100644 --- a/pwsh/dev/functions/getPolicyHash.ps1 +++ b/pwsh/dev/functions/getPolicyHash.ps1 @@ -1,9 +1,8 @@ -function getPolicyHash { - [CmdletBinding()] +function getPolicyHash { param ( [Parameter(Mandatory)] [string] $json ) - return [System.BitConverter]::ToString([System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($json))) + return [string]([System.BitConverter]::ToString([System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($json)))) } \ No newline at end of file diff --git a/pwsh/dev/functions/getPolicyRemediation.ps1 b/pwsh/dev/functions/getPolicyRemediation.ps1 index 1d5f3d39..b7766886 100644 --- a/pwsh/dev/functions/getPolicyRemediation.ps1 +++ b/pwsh/dev/functions/getPolicyRemediation.ps1 @@ -1,4 +1,4 @@ -function getPolicyRemediation { +function getPolicyRemediation { $currentTask = 'Getting NonCompliant (dine/modify)' Write-Host $currentTask #ref: https://learn.microsoft.com/en-us/rest/api/azureresourcegraph/resourcegraph/resources/resources diff --git a/pwsh/dev/functions/getPrivateEndpointCapableResourceTypes.ps1 b/pwsh/dev/functions/getPrivateEndpointCapableResourceTypes.ps1 index 1dee74b3..ea1c32ae 100644 --- a/pwsh/dev/functions/getPrivateEndpointCapableResourceTypes.ps1 +++ b/pwsh/dev/functions/getPrivateEndpointCapableResourceTypes.ps1 @@ -1,4 +1,4 @@ -function getPrivateEndpointCapableResourceTypes { +function getPrivateEndpointCapableResourceTypes { $startGetAvailablePrivateEndpointTypes = Get-Date $privateEndpointAvailabilityCheckCompleted = $false $subsToProcessForGettingPrivateEndpointTypes = [System.Collections.ArrayList]@() diff --git a/pwsh/dev/functions/getResourceDiagnosticsCapability.ps1 b/pwsh/dev/functions/getResourceDiagnosticsCapability.ps1 index 727ffc0b..10a9b2d8 100644 --- a/pwsh/dev/functions/getResourceDiagnosticsCapability.ps1 +++ b/pwsh/dev/functions/getResourceDiagnosticsCapability.ps1 @@ -1,4 +1,4 @@ -function getResourceDiagnosticsCapability { +function getResourceDiagnosticsCapability { Write-Host 'Checking Resource Types Diagnostics capability (1st party only)' $startResourceDiagnosticsCheck = Get-Date if (($resourcesAll).count -gt 0) { diff --git a/pwsh/dev/functions/getSubscriptions.ps1 b/pwsh/dev/functions/getSubscriptions.ps1 index cf182a59..f7ef41eb 100644 --- a/pwsh/dev/functions/getSubscriptions.ps1 +++ b/pwsh/dev/functions/getSubscriptions.ps1 @@ -1,4 +1,4 @@ -function getSubscriptions { +function getSubscriptions { $startGetSubscriptions = Get-Date $currentTask = 'Getting all Subscriptions' Write-Host "$currentTask" diff --git a/pwsh/dev/functions/getTenantDetails.ps1 b/pwsh/dev/functions/getTenantDetails.ps1 index cf2b1214..58f19420 100644 --- a/pwsh/dev/functions/getTenantDetails.ps1 +++ b/pwsh/dev/functions/getTenantDetails.ps1 @@ -1,4 +1,4 @@ -function getTenantDetails { +function getTenantDetails { $currentTask = 'Get Tenant details' Write-Host $currentTask $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/tenants?api-version=2020-01-01" diff --git a/pwsh/dev/functions/handleCloudEnvironment.ps1 b/pwsh/dev/functions/handleCloudEnvironment.ps1 index 32aee825..8cd3e81f 100644 --- a/pwsh/dev/functions/handleCloudEnvironment.ps1 +++ b/pwsh/dev/functions/handleCloudEnvironment.ps1 @@ -1,4 +1,4 @@ -function handleCloudEnvironment { +function handleCloudEnvironment { Write-Host "Environment: $($azAPICallConf['checkContext'].Environment.Name)" if ($DoAzureConsumption) { if ($azAPICallConf['checkContext'].Environment.Name -eq 'AzureChinaCloud') { diff --git a/pwsh/dev/functions/html/htmlFunctions.ps1 b/pwsh/dev/functions/html/htmlFunctions.ps1 index b8fef36c..8fdb5367 100644 --- a/pwsh/dev/functions/html/htmlFunctions.ps1 +++ b/pwsh/dev/functions/html/htmlFunctions.ps1 @@ -1,4 +1,4 @@ -#region HTML +#region HTML function HierarchyMgHTML($mgChild) { $mgDetails = $htMgDetails.($mgChild).details $mgName = $mgDetails.mgName diff --git a/pwsh/dev/functions/namingValidation.ps1 b/pwsh/dev/functions/namingValidation.ps1 index 7c39c788..f3147448 100644 --- a/pwsh/dev/functions/namingValidation.ps1 +++ b/pwsh/dev/functions/namingValidation.ps1 @@ -1,4 +1,4 @@ -function NamingValidation($toCheck) { +function NamingValidation($toCheck) { $checks = @(':', '/', '\', '<', '>', '|', '"') $array = [System.Collections.ArrayList]@() foreach ($check in $checks) { diff --git a/pwsh/dev/functions/prepareData.ps1 b/pwsh/dev/functions/prepareData.ps1 index 1f0ad2be..45056dab 100644 --- a/pwsh/dev/functions/prepareData.ps1 +++ b/pwsh/dev/functions/prepareData.ps1 @@ -1,4 +1,4 @@ -function prepareData { +function prepareData { Write-Host 'Preparing Data' $startPreparingArrays = Get-Date $script:optimizedTableForPathQuery = ($newTable | Select-Object -Property level, mg*, subscription*) | Sort-Object -Property level, mgid, subscriptionId -Unique diff --git a/pwsh/dev/functions/processAADGroups.ps1 b/pwsh/dev/functions/processAADGroups.ps1 index bfc55c3b..206b9abf 100644 --- a/pwsh/dev/functions/processAADGroups.ps1 +++ b/pwsh/dev/functions/processAADGroups.ps1 @@ -1,4 +1,4 @@ -function processAADGroups { +function processAADGroups { if ($NoPIMEligibility) { Write-Host 'Resolving Microsoft Entra groups (for which a RBAC role assignment exists)' } diff --git a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 index dffb8202..9763b691 100644 --- a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 +++ b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 @@ -1,4 +1,4 @@ -function processALZPolicyVersionChecker { +function processALZPolicyVersionChecker { $start = Get-Date Write-Host "Processing 'Azure Landing Zones (ALZ) Policy Version Checker' base data" $ALZRepositoryURI = 'https://github.com/Azure/Enterprise-Scale.git' @@ -51,6 +51,11 @@ function processALZPolicyVersionChecker { Write-Host " Switching to directory '$($ALZPath)/Enterprise-Scale'" Set-Location "$($ALZPath)/Enterprise-Scale" + #devSkim ... + $ALZCommitIdP1 = '3476914f9ba9a8f3f641a' + $ALZCommitIdP2 = '25497dfb24a4efa1017' + $ALZCommitId = "$($ALZCommitIdP1)$($ALZCommitIdP2)" + $allESLZPolicies = @{} $allESLZPolicySets = @{} $allESLZPolicyHashes = @{} @@ -62,7 +67,7 @@ function processALZPolicyVersionChecker { $processDataPolicies = $true foreach ($commit in $gitHist | Sort-Object -Property Date) { if ($processDataPolicies) { - if ($commit.CommitId -eq '3476914f9ba9a8f3f641a25497dfb24a4efa1017') { + if ($commit.CommitId -eq $ALZCommitId) { $processDataPolicies = $false continue } @@ -133,7 +138,7 @@ function processALZPolicyVersionChecker { $doNewALZPolicyReadingApproach = $false foreach ($commit in $gitHist | Sort-Object -Property Date) { - if ($commit.CommitId -eq '3476914f9ba9a8f3f641a25497dfb24a4efa1017') { + if ($commit.CommitId -eq $ALZCommitId) { $doNewALZPolicyReadingApproach = $true } #Write-Host "processing commit $($commit.CommitId) - doNewALZPolicyReadingApproach: $doNewALZPolicyReadingApproach" diff --git a/pwsh/dev/functions/processApplications.ps1 b/pwsh/dev/functions/processApplications.ps1 index 818a3f5b..dc190887 100644 --- a/pwsh/dev/functions/processApplications.ps1 +++ b/pwsh/dev/functions/processApplications.ps1 @@ -1,4 +1,4 @@ -function processApplications { +function processApplications { Write-Host 'Processing Service Principals - Applications' $script:servicePrincipalsOfTypeApplication = $htServicePrincipals.Keys.where( { $htServicePrincipals.($_).servicePrincipalType -eq 'Application' -and $htServicePrincipals.($_).appOwnerOrganizationId -eq $azAPICallConf['checkContext'].Subscription.TenantId } ) if ($azAPICallConf['htParameters'].userType -eq 'Guest') { diff --git a/pwsh/dev/functions/processDataCollection.ps1 b/pwsh/dev/functions/processDataCollection.ps1 index 40227e0f..35824935 100644 --- a/pwsh/dev/functions/processDataCollection.ps1 +++ b/pwsh/dev/functions/processDataCollection.ps1 @@ -1,4 +1,4 @@ -function processDataCollection { +function processDataCollection { [CmdletBinding()]Param( [string]$mgId ) diff --git a/pwsh/dev/functions/processDefinitionInsights.ps1 b/pwsh/dev/functions/processDefinitionInsights.ps1 index 4bf83c9f..9cd54fa5 100644 --- a/pwsh/dev/functions/processDefinitionInsights.ps1 +++ b/pwsh/dev/functions/processDefinitionInsights.ps1 @@ -1,8 +1,8 @@ -function processDefinitionInsights() { +function processDefinitionInsights() { $startDefinitionInsights = Get-Date Write-Host ' Building DefinitionInsights' - $md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider + $SHA256 = New-Object -TypeName System.Security.Cryptography.SHA256CryptoServiceProvider $utf8 = New-Object -TypeName System.Text.UTF8Encoding #region definitionInsightsAzurePolicy @@ -275,7 +275,7 @@ function processDefinitionInsights() { } $json = $($policy.Json | ConvertTo-Json -Depth 99) - $guid = ([System.BitConverter]::ToString($md5.ComputeHash($utf8.GetBytes($policy.PolicyDefinitionId)))) -replace '-' + $guid = ([System.BitConverter]::ToString($SHA256.ComputeHash($utf8.GetBytes($policy.PolicyDefinitionId)))) -replace '-' @" @@ -564,7 +564,7 @@ tf.init();}} $scopeDetails = "$($policySet.ScopeId) ($($htEntities.($policySet.ScopeId).DisplayName))" } $json = $($policySet.Json | ConvertTo-Json -Depth 99) - $guid = ([System.BitConverter]::ToString($md5.ComputeHash($utf8.GetBytes($policySet.PolicyDefinitionId)))) -replace '-' + $guid = ([System.BitConverter]::ToString($SHA256.ComputeHash($utf8.GetBytes($policySet.PolicyDefinitionId)))) -replace '-' @" diff --git a/pwsh/dev/functions/processDiagramMermaid.ps1 b/pwsh/dev/functions/processDiagramMermaid.ps1 index e4dbf3db..b953d757 100644 --- a/pwsh/dev/functions/processDiagramMermaid.ps1 +++ b/pwsh/dev/functions/processDiagramMermaid.ps1 @@ -1,4 +1,4 @@ -function processDiagramMermaid() { +function processDiagramMermaid() { if ($ManagementGroupId -ne $azAPICallConf['checkContext'].Tenant.Id) { $optimizedTableForPathQueryMg = $optimizedTableForPathQueryMg.where({ $_.mgParentId -ne "'upperScopes'" }) } diff --git a/pwsh/dev/functions/processHierarchyMapOnly.ps1 b/pwsh/dev/functions/processHierarchyMapOnly.ps1 index 534716b3..db9bcc39 100644 --- a/pwsh/dev/functions/processHierarchyMapOnly.ps1 +++ b/pwsh/dev/functions/processHierarchyMapOnly.ps1 @@ -1,4 +1,4 @@ -function processHierarchyMapOnly { +function processHierarchyMapOnly { foreach ($entity in $htEntities.values) { if ($entity.parentNameChain -contains $ManagementGroupID -or $entity.Id -eq $ManagementGroupId) { diff --git a/pwsh/dev/functions/processHierarchyMapOnlyCustomData.ps1 b/pwsh/dev/functions/processHierarchyMapOnlyCustomData.ps1 index 0515eca8..3e0557a6 100644 --- a/pwsh/dev/functions/processHierarchyMapOnlyCustomData.ps1 +++ b/pwsh/dev/functions/processHierarchyMapOnlyCustomData.ps1 @@ -1,4 +1,4 @@ -function processHierarchyMapOnlyCustomData { +function processHierarchyMapOnlyCustomData { Write-Host 'HierarchyMapOnly with custom data' -ForegroundColor Yellow Write-Host ' Parameter HierarchyMapOnly:' $HierarchyMapOnly Write-Host ' Check if HierarchyMapOnlyCustomDataJSON is valid JSON' diff --git a/pwsh/dev/functions/processManagedIdentities.ps1 b/pwsh/dev/functions/processManagedIdentities.ps1 index 1d68e1ce..66d1d109 100644 --- a/pwsh/dev/functions/processManagedIdentities.ps1 +++ b/pwsh/dev/functions/processManagedIdentities.ps1 @@ -1,4 +1,4 @@ -function processManagedIdentities { +function processManagedIdentities { Write-Host 'Processing Service Principals - Managed Identities' $startSPMI = Get-Date $script:servicePrincipalsOfTypeManagedIdentity = $htServicePrincipals.Keys.where( { $htServicePrincipals.($_).servicePrincipalType -eq 'ManagedIdentity' } ) diff --git a/pwsh/dev/functions/processNetwork.ps1 b/pwsh/dev/functions/processNetwork.ps1 index 42d9a4e7..fa2b0d19 100644 --- a/pwsh/dev/functions/processNetwork.ps1 +++ b/pwsh/dev/functions/processNetwork.ps1 @@ -1,4 +1,4 @@ -function processNetwork { +function processNetwork { $start = Get-Date Write-Host "Processing Network enrichment ($($arrayVNets.Count) Virtual Networks)" diff --git a/pwsh/dev/functions/processPrivateEndpoints.ps1 b/pwsh/dev/functions/processPrivateEndpoints.ps1 index d73c7bad..4d723a8c 100644 --- a/pwsh/dev/functions/processPrivateEndpoints.ps1 +++ b/pwsh/dev/functions/processPrivateEndpoints.ps1 @@ -1,4 +1,4 @@ -function processPrivateEndpoints { +function processPrivateEndpoints { $start = Get-Date Write-Host 'Processing Private Endpoints enrichment' diff --git a/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 b/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 index fa5784c5..07daddbe 100644 --- a/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 +++ b/pwsh/dev/functions/processScopeInsightsMgOrSub.ps1 @@ -1,4 +1,4 @@ -function processScopeInsightsMgOrSub($mgOrSub, $mgChild, $subscriptionId, $subscriptionsMgId) { +function processScopeInsightsMgOrSub($mgOrSub, $mgChild, $subscriptionId, $subscriptionsMgId) { $script:scopescnter++ $htmlScopeInsights = $null $htmlScopeInsights = [System.Text.StringBuilder]::new() @@ -2302,7 +2302,8 @@ paging: {results_per_page: ['Records: ', [$spectrum]]},/*state: {types: ['local_ $allPSRuleResultsUnderThisMg = [system.collections.ArrayList]@() foreach ($mg in $grpPSRuleManagementGroups) { - if ($htManagementGroupsMgPath.($mg.name -replace '.*/').path -contains $mgchild) { + $mgNameIdHlper = $mg.name -replace '.*/' + if ($htManagementGroupsMgPath.($mgNameIdHlper).path -contains $mgchild) { $allPSRuleResultsUnderThisMg.AddRange($mg.Group) } } @@ -3930,7 +3931,8 @@ btn_reset: true, highlight_keywords: true, alternate_rows: true, auto_filter: { if (-not $NoScopeInsights) { if ($scopescnter % 50 -eq 0) { $script:scopescnter = 0 - Write-Host ' append file duration: '(Measure-Command { $script:html | Add-Content -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName).html" -Encoding utf8 -Force }).TotalSeconds 'seconds' + $addContentDurationInSeconds = (Measure-Command { $script:html | Add-Content -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName).html" -Encoding utf8 -Force }).TotalSeconds + Write-Host " append file duration: $addContentDurationInSeconds seconds" $script:html = $null } } diff --git a/pwsh/dev/functions/processStorageAccountAnalysis.ps1 b/pwsh/dev/functions/processStorageAccountAnalysis.ps1 index 23bb5585..b8aceba0 100644 --- a/pwsh/dev/functions/processStorageAccountAnalysis.ps1 +++ b/pwsh/dev/functions/processStorageAccountAnalysis.ps1 @@ -1,4 +1,4 @@ -function processStorageAccountAnalysis { +function processStorageAccountAnalysis { $start = Get-Date Write-Host 'Processing Storage Account Analysis' $storageAccountsCount = $storageAccounts.count diff --git a/pwsh/dev/functions/processTenantSummary.ps1 b/pwsh/dev/functions/processTenantSummary.ps1 index 27d736f5..bde73e4e 100644 --- a/pwsh/dev/functions/processTenantSummary.ps1 +++ b/pwsh/dev/functions/processTenantSummary.ps1 @@ -1,4 +1,4 @@ -function processTenantSummary() { +function processTenantSummary() { Write-Host ' Building TenantSummary' showMemoryUsage if ($getMgParentName -eq 'Tenant Root') { @@ -958,16 +958,17 @@ function processTenantSummary() { if (($tenantPolicy.RoleDefinitionIds) -ne 'n/a') { $policyRoleDefinitionsArray = @() $policyRoleDefinitionsArray = foreach ($roleDefinitionId in $tenantPolicy.RoleDefinitionIds | Sort-Object) { - if (($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').LinkToAzAdvertizer) { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').LinkToAzAdvertizer + $roleDefinitionIdGuid = $roledefinitionId -replace '.*/' + if (($htCacheDefinitionsRole).($roleDefinitionIdGuid).LinkToAzAdvertizer) { + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).LinkToAzAdvertizer } else { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name -replace '<', '<' -replace '>', '>' + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name -replace '<', '<' -replace '>', '>' } } $policyRoleDefinitionsClearArray = @() $policyRoleDefinitionsClearArray = foreach ($roleDefinitionId in $tenantPolicy.RoleDefinitionIds | Sort-Object) { - ($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name + ($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name } $policyRoleDefinitions = $policyRoleDefinitionsArray -join "$CsvDelimiterOpposite " $policyRoleDefinitionsClear = $policyRoleDefinitionsClearArray -join "$CsvDelimiterOpposite " @@ -3797,8 +3798,9 @@ extensions: [{ name: 'sort' }] $hlp = $htPolicyAssignmentRelatedRoleAssignments.($policyAssignmentAll.PolicyAssignmentId) $relatedRoleAssignments = $hlp.relatedRoleAssignments $relatedRoleAssignmentsClear = $hlp.relatedRoleAssignmentsClear - if ($htManagedIdentityDisplayName.("$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)")) { - $hlp = $htManagedIdentityDisplayName.("$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)") + $hlperVar = "$($policyAssignmentAll.PolicyAssignmentId -replace '.*/')_$($policyAssignmentAll.PolicyAssignmentId)" + if ($htManagedIdentityDisplayName.($hlperVar)) { + $hlp = $htManagedIdentityDisplayName.($hlperVar) $policyAssignmentMI = "$($hlp.displayname) (SPObjId: $($hlp.id))" } } @@ -11032,8 +11034,9 @@ extensions: [{ name: 'sort' }] $roleDefinitionIdsArray = [System.Collections.ArrayList]@() foreach ($roleDefinitionId in ($policy).Json.properties.policyrule.then.details.roleDefinitionIds) { - if (($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/')) { - $null = $roleDefinitionIdsArray.Add("$(($htCacheDefinitionsRole).($roleDefinitionId -replace '.*/').Name) ($($roleDefinitionId -replace '.*/'))") + $roleDefinitionIdGuid = $roleDefinitionId -replace '.*/' + if (($htCacheDefinitionsRole).($roleDefinitionIdGuid)) { + $null = $roleDefinitionIdsArray.Add("$(($htCacheDefinitionsRole).($roleDefinitionIdGuid).Name) ($($roleDefinitionIdGuid))") } else { Write-Host " DiagnosticsLifeCycle: unknown RoleDefinition '$roleDefinitionId'" diff --git a/pwsh/dev/functions/removeInvalidFileNameChars.ps1 b/pwsh/dev/functions/removeInvalidFileNameChars.ps1 index 7712dca5..1b305af4 100644 --- a/pwsh/dev/functions/removeInvalidFileNameChars.ps1 +++ b/pwsh/dev/functions/removeInvalidFileNameChars.ps1 @@ -1,11 +1,10 @@ -function removeInvalidFileNameChars { +function removeInvalidFileNameChars { param( - [Parameter(Mandatory = $true, - Position = 0, - ValueFromPipeline = $true, - ValueFromPipelineByPropertyName = $true)] - [String]$Name + [Parameter(Mandatory = $true)] + [string] + $Name ) + if ($Name -like '`[Deprecated`]:*') { $Name = $Name -replace '\[Deprecated\]\:', '[Deprecated]' } @@ -15,5 +14,6 @@ function removeInvalidFileNameChars { if ($Name -like '`[ASC Private Preview`]:*') { $Name = $Name -replace '\[ASC Private Preview\]\:', '[ASC Private Preview]' } + return ($Name -replace ':', '_' -replace '/', '_' -replace '\\', '_' -replace '<', '_' -replace '>', '_' -replace '\*', '_' -replace '\?', '_' -replace '\|', '_' -replace '"', '_') } \ No newline at end of file diff --git a/pwsh/dev/functions/resolveObjectIds.ps1 b/pwsh/dev/functions/resolveObjectIds.ps1 index 00af7fe2..a0120957 100644 --- a/pwsh/dev/functions/resolveObjectIds.ps1 +++ b/pwsh/dev/functions/resolveObjectIds.ps1 @@ -1,4 +1,4 @@ -function ResolveObjectIds { +function ResolveObjectIds { [CmdletBinding()]Param( [object] $objectIds, diff --git a/pwsh/dev/functions/runInfo.ps1 b/pwsh/dev/functions/runInfo.ps1 index 4f0bceb2..8664722b 100644 --- a/pwsh/dev/functions/runInfo.ps1 +++ b/pwsh/dev/functions/runInfo.ps1 @@ -1,4 +1,4 @@ -function runInfo { +function runInfo { #region RunInfo Write-Host 'Run Info:' if ($HierarchyMapOnly) { diff --git a/pwsh/dev/functions/selectMg.ps1 b/pwsh/dev/functions/selectMg.ps1 index 02984a90..f6357004 100644 --- a/pwsh/dev/functions/selectMg.ps1 +++ b/pwsh/dev/functions/selectMg.ps1 @@ -1,4 +1,4 @@ -function selectMg() { +function selectMg() { Write-Host 'Please select a Management Group from the list below:' $MgtGroupArray | Select-Object '#', Name, @{Name = 'displayName'; Expression = { $_.properties.displayName } }, Id | Format-Table Write-Host "If you don't see your ManagementGroupID try using the parameter -ManagementGroupID" -ForegroundColor Yellow diff --git a/pwsh/dev/functions/setBaseVariablesMG.ps1 b/pwsh/dev/functions/setBaseVariablesMG.ps1 index 952a5cdd..39960d1d 100644 --- a/pwsh/dev/functions/setBaseVariablesMG.ps1 +++ b/pwsh/dev/functions/setBaseVariablesMG.ps1 @@ -1,4 +1,4 @@ -function setBaseVariablesMG { +function setBaseVariablesMG { if (($azAPICallConf['checkContext']).Tenant.Id -ne $ManagementGroupId) { $script:mgSubPathTopMg = $selectedManagementGroupId.ParentName $script:getMgParentId = $selectedManagementGroupId.ParentName diff --git a/pwsh/dev/functions/setOutput.ps1 b/pwsh/dev/functions/setOutput.ps1 index 3f141cb3..47eda1b9 100644 --- a/pwsh/dev/functions/setOutput.ps1 +++ b/pwsh/dev/functions/setOutput.ps1 @@ -1,4 +1,4 @@ -function setOutput { +function setOutput { if (-not [IO.Path]::IsPathRooted($outputPath)) { $outputPath = Join-Path -Path (Get-Location).Path -ChildPath $outputPath } diff --git a/pwsh/dev/functions/setTranscript.ps1 b/pwsh/dev/functions/setTranscript.ps1 index 2544390e..118101d4 100644 --- a/pwsh/dev/functions/setTranscript.ps1 +++ b/pwsh/dev/functions/setTranscript.ps1 @@ -1,4 +1,4 @@ -function setTranscript { +function setTranscript { if ($ManagementGroupId) { if ($onAzureDevOpsOrGitHubActions -eq $true) { if ($HierarchyMapOnly -eq $true) { diff --git a/pwsh/dev/functions/showMemoryUsage.ps1 b/pwsh/dev/functions/showMemoryUsage.ps1 index b86dd7e7..bd6ec36c 100644 --- a/pwsh/dev/functions/showMemoryUsage.ps1 +++ b/pwsh/dev/functions/showMemoryUsage.ps1 @@ -1,4 +1,4 @@ -function showMemoryUsage { +function showMemoryUsage { function makeDouble { [CmdletBinding()] diff --git a/pwsh/dev/functions/stats.ps1 b/pwsh/dev/functions/stats.ps1 index 0bd487fc..c0d59890 100644 --- a/pwsh/dev/functions/stats.ps1 +++ b/pwsh/dev/functions/stats.ps1 @@ -1,4 +1,4 @@ -function stats { +function stats { #region Stats if (-not $StatsOptOut) { diff --git a/pwsh/dev/functions/testGuid.ps1 b/pwsh/dev/functions/testGuid.ps1 index 8a4b7607..1cff3905 100644 --- a/pwsh/dev/functions/testGuid.ps1 +++ b/pwsh/dev/functions/testGuid.ps1 @@ -1,4 +1,4 @@ -function testGuid { +function testGuid { [OutputType([bool])] param ( diff --git a/pwsh/dev/functions/validateAccess.ps1 b/pwsh/dev/functions/validateAccess.ps1 index 9d699ad5..2ee7ba8d 100644 --- a/pwsh/dev/functions/validateAccess.ps1 +++ b/pwsh/dev/functions/validateAccess.ps1 @@ -1,4 +1,4 @@ -function validateAccess { +function validateAccess { #region validationAccess #validation / check 'Microsoft Graph API' Access $permissionCheckResults = @() diff --git a/pwsh/dev/functions/validateLeastPrivilegeForUser.ps1 b/pwsh/dev/functions/validateLeastPrivilegeForUser.ps1 index faf5445f..839f6dc4 100644 --- a/pwsh/dev/functions/validateLeastPrivilegeForUser.ps1 +++ b/pwsh/dev/functions/validateLeastPrivilegeForUser.ps1 @@ -1,4 +1,4 @@ -function validateLeastPrivilegeForUser { +function validateLeastPrivilegeForUser { $currentTask = "Validate least priviledge (Azure Resource side) for executing user $($azapicallConf['htParameters'].userObjectId)" $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.Authorization/roleAssignments?api-version=2022-04-01&`$filter=principalId eq '$($azapicallConf['htParameters'].userObjectId)'" $method = 'GET' @@ -12,7 +12,7 @@ function validateLeastPrivilegeForUser { $currentTask = "Get RBAC Role definition '$nonReaderRoleAssigned'" $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($nonReaderRoleAssigned)?api-version=2022-04-01" $method = 'GET' - $getRole = AzAPICall -AzAPICallConfiguration $azapicallConf -uri $uri -listenOn Content + $getRole = AzAPICall -AzAPICallConfiguration $azapicallConf -uri $uri -method $method -listenOn Content if ($getRole.properties.roleName -eq 'owner' -or $getRole.properties.roleName -eq 'contributor') { Write-Host " - $($getRole.properties.roleName) ($($getRole.properties.type)) !!!" diff --git a/pwsh/dev/functions/verifyModules3rd.ps1 b/pwsh/dev/functions/verifyModules3rd.ps1 index d5f99402..21683b5e 100644 --- a/pwsh/dev/functions/verifyModules3rd.ps1 +++ b/pwsh/dev/functions/verifyModules3rd.ps1 @@ -1,4 +1,4 @@ -function verifyModules3rd { +function verifyModules3rd { [CmdletBinding()]Param( [object]$modules ) diff --git a/pwsh/prerequisites.ps1 b/pwsh/prerequisites.ps1 index 146530ce..65a74ebb 100644 --- a/pwsh/prerequisites.ps1 +++ b/pwsh/prerequisites.ps1 @@ -1,4 +1,4 @@ -#20220521_1 +#20220521_1 #This script should be run in Azure DevOps Pipelines and GitHub Actions only param( diff --git a/version.json b/version.json index 5e01716e..13ebe353 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.5" + "ProductVersion": "6.4.6" } \ No newline at end of file From db843c59bca8291b8cb60f54e671b251961d9873 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Wed, 15 May 2024 20:25:26 +0200 Subject: [PATCH 02/20] devSkim --- .github/workflows/devskim.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/devskim.yml diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml new file mode 100644 index 00000000..d411e173 --- /dev/null +++ b/.github/workflows/devskim.yml @@ -0,0 +1,34 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: DevSkim + +on: + push: + branches: [ "developmentJH" ] + pull_request: + branches: [ "developmentJH" ] + # schedule: + # - cron: '28 13 * * 2' + +jobs: + lint: + name: DevSkim + runs-on: ubuntu-20.04 + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run DevSkim scanner + uses: microsoft/DevSkim-Action@v1 + + - name: Upload DevSkim scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: devskim-results.sarif \ No newline at end of file From ac951cc27688db40c3d79cd799bb7f47b453c01a Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Wed, 15 May 2024 20:29:32 +0200 Subject: [PATCH 03/20] psScriptAnalyzer --- .github/workflows/psScriptAnalyzer.yml | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/psScriptAnalyzer.yml diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml new file mode 100644 index 00000000..41425a33 --- /dev/null +++ b/.github/workflows/psScriptAnalyzer.yml @@ -0,0 +1,50 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# +# https://github.com/microsoft/action-psscriptanalyzer +# For more information on PSScriptAnalyzer in general, see +# https://github.com/PowerShell/PSScriptAnalyzer + +name: PSScriptAnalyzer + +on: + push: + branches: [ "developmentJH" ] + pull_request: + branches: [ "developmentJH" ] + # schedule: + # - cron: '25 6 * * 1' + +permissions: + contents: read + +jobs: + build: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: PSScriptAnalyzer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run PSScriptAnalyzer + uses: microsoft/psscriptanalyzer-action@6b2948b1944407914a58661c49941824d149734f + with: + # Check https://github.com/microsoft/action-psscriptanalyzer for more info about the options. + # The below set up runs PSScriptAnalyzer to your entire repository and runs some basic security rules. + path: .\ + recurse: true + # Include your own basic security rules. Removing this option will run all the rules + # includeRule: '"PSAvoidGlobalAliases", "PSAvoidUsingConvertToSecureStringWithPlainText"' + excludeRule: '"PSAvoidUsingWriteHost", "PSUseDeclaredVarsMoreThanAssignments", "PSReviewUnusedParameter"' + output: results.sarif + + # Upload the SARIF file generated in the previous step + - name: Upload SARIF results file + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: results.sarif \ No newline at end of file From eddcfcb27004e842c71bb9dec5613842cefbfb17 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 16 May 2024 19:34:59 +0200 Subject: [PATCH 04/20] condition devSkim,pssa & minor --- .github/workflows/devskim.yml | 1 + .github/workflows/psScriptAnalyzer.yml | 1 + pwsh/AzGovVizParallel.ps1 | 3 ++- pwsh/dev/devAzGovVizParallel.ps1 | 2 +- pwsh/dev/functions/getOrphanedResources.ps1 | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml index d411e173..aa7268d6 100644 --- a/.github/workflows/devskim.yml +++ b/.github/workflows/devskim.yml @@ -14,6 +14,7 @@ on: # - cron: '28 13 * * 2' jobs: + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' lint: name: DevSkim runs-on: ubuntu-20.04 diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index 41425a33..ee4982ba 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -21,6 +21,7 @@ permissions: contents: read jobs: + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' build: permissions: contents: read # for actions/checkout to fetch code diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index b2a6665f..94b939f0 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -388,7 +388,7 @@ Param # AzAPICall related parameters ---> [string] - $ARMLocation = 'westeurope', + $ARMLocation = 'northeurope', [string] $ScriptPath = 'pwsh', #e.g. 'myfolder\pwsh' @@ -3807,6 +3807,7 @@ resources | where type =~ 'microsoft.network/publicIpAddresses' resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' +| where not(name endswith "-asr") | project type, subscriptionId, Resource=id, Intent='$intent' "@ intent = $intent diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index d9f5c8e2..478e8bf5 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -388,7 +388,7 @@ Param # AzAPICall related parameters ---> [string] - $ARMLocation = 'westeurope', + $ARMLocation = 'northeurope', [string] $ScriptPath = 'pwsh', #e.g. 'myfolder\pwsh' diff --git a/pwsh/dev/functions/getOrphanedResources.ps1 b/pwsh/dev/functions/getOrphanedResources.ps1 index 4bdee614..bb928eee 100644 --- a/pwsh/dev/functions/getOrphanedResources.ps1 +++ b/pwsh/dev/functions/getOrphanedResources.ps1 @@ -114,6 +114,7 @@ resources | where type =~ 'microsoft.network/publicIpAddresses' resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' +| where not(name endswith "-asr") | project type, subscriptionId, Resource=id, Intent='$intent' "@ intent = $intent From 432d6243f21579ee1a21ab683f9d0b6fa7d9dfbf Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 16 May 2024 19:39:51 +0200 Subject: [PATCH 05/20] fix --- .github/workflows/devskim.yml | 2 +- .github/workflows/psScriptAnalyzer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml index aa7268d6..917d4030 100644 --- a/.github/workflows/devskim.yml +++ b/.github/workflows/devskim.yml @@ -14,8 +14,8 @@ on: # - cron: '28 13 * * 2' jobs: - if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' lint: + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' name: DevSkim runs-on: ubuntu-20.04 permissions: diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index ee4982ba..5c78c84d 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -21,8 +21,8 @@ permissions: contents: read jobs: - if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' build: + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results From 35b4c8ef2a664e75553c1a09d0cf39920ab41fa6 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 16 May 2024 19:40:33 +0200 Subject: [PATCH 06/20] fix --- .github/workflows/psScriptAnalyzer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index 5c78c84d..4c7d852d 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -22,7 +22,7 @@ permissions: jobs: build: - if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results From 23b9b6ccc0ecebf86733f9db21700f6ff1fa96d8 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 16 May 2024 19:42:29 +0200 Subject: [PATCH 07/20] test condition --- .github/workflows/devskim.yml | 2 +- .github/workflows/psScriptAnalyzer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml index 917d4030..f29b0eb1 100644 --- a/.github/workflows/devskim.yml +++ b/.github/workflows/devskim.yml @@ -15,7 +15,7 @@ on: jobs: lint: - if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' + if: github.repository == 'NOTJulianHayward/Azure-MG-Sub-Governance-Reporting' name: DevSkim runs-on: ubuntu-20.04 permissions: diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index 4c7d852d..aee7661e 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -22,7 +22,7 @@ permissions: jobs: build: - if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' + if: github.repository == 'NOTJulianHayward/Azure-MG-Sub-Governance-Reporting' permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results From ca49d4e3bc1c1fd023077cae2376efaf12748fc3 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 16 May 2024 19:43:30 +0200 Subject: [PATCH 08/20] devSkim&pssa gtg --- .github/workflows/devskim.yml | 2 +- .github/workflows/psScriptAnalyzer.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml index f29b0eb1..917d4030 100644 --- a/.github/workflows/devskim.yml +++ b/.github/workflows/devskim.yml @@ -15,7 +15,7 @@ on: jobs: lint: - if: github.repository == 'NOTJulianHayward/Azure-MG-Sub-Governance-Reporting' + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' name: DevSkim runs-on: ubuntu-20.04 permissions: diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index aee7661e..4c7d852d 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -22,7 +22,7 @@ permissions: jobs: build: - if: github.repository == 'NOTJulianHayward/Azure-MG-Sub-Governance-Reporting' + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results From bf848f23ad3a413b7e7b7400ba23b3bfc91b3ff2 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 23 May 2024 14:16:04 +0200 Subject: [PATCH 09/20] 6.4.7 --- pwsh/AzGovVizParallel.ps1 | 57 ++++++++++++++----- pwsh/dev/devAzGovVizParallel.ps1 | 23 +++++++- pwsh/dev/functions/addHtParameters.ps1 | 1 + pwsh/dev/functions/cacheBuiltIn.ps1 | 10 ++-- .../dataCollectionFunctions.ps1 | 9 ++- pwsh/dev/functions/getConsumption.ps1 | 12 ++-- pwsh/dev/functions/getOrphanedResources.ps1 | 2 +- version.json | 2 +- 8 files changed, 87 insertions(+), 29 deletions(-) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 94b939f0..5f707e83 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.6', + $ProductVersion = '6.4.7', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -627,7 +627,26 @@ Param $MSTenantIds = @('2f4a9838-26b7-47ee-be60-ccc1fdec5953', '33e01921-4d64-4f8c-a055-5bdaffd5e33d'), [array] - $ValidPolicyEffects = @('addToNetworkGroup', 'append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting', 'mutate') + $ValidPolicyEffects = @('addToNetworkGroup', 'append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting', 'mutate'), + + [hashtable] + $APIMappingCloudEnvironment = @{ + roleDefinitions = @{ + AzureCloud = '2023-07-01-preview' + AzureUSGovernment = '2022-05-01-preview' + AzureChinaCloud = '2022-05-01-preview' + } + costManagementQuery = @{ + AzureCloud = '2024-01-01' + AzureUSGovernment = '2023-09-01' + AzureChinaCloud = '2023-09-01' + } + securityPricings = @{ + AzureCloud = '2024-01-01' + AzureUSGovernment = '2023-01-01' + AzureChinaCloud = '2023-01-01' + } + } ) $Error.clear() @@ -690,6 +709,7 @@ function addHtParameters { GitHubActionsOIDC = [bool]$GitHubActionsOIDC NoNetwork = [bool]$NoNetwork ThrottleLimit = $ThrottleLimit + APIMappingCloudEnvironment = $APIMappingCloudEnvironment } Write-Host 'htParameters:' $azAPICallConf['htParameters'] | Format-Table -AutoSize | Out-String @@ -2178,16 +2198,18 @@ function cacheBuiltIn { if ($builtInCapability -eq 'RoleDefinitions') { + $roledefinitionsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.roledefinitions.($azAPICallConf['htParameters'].azureCloudEnvironment) + #region subscriptionScope if ($ignoreARMLocation) { $currentTask = 'Caching built-in Role definitions (subscriptionScope)' Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } else { $currentTask = "Caching built-in Role definitions (Location: '$($ARMLocation)') (subscriptionScope)" Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } $method = 'GET' @@ -2239,12 +2261,12 @@ function cacheBuiltIn { if ($ignoreARMLocation) { $currentTask = 'Caching built-in Role definitions (tenantScope)' Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } else { $currentTask = "Caching built-in Role definitions (Location: '$($ARMLocation)') (tenantScope)" Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } $method = 'GET' @@ -2704,6 +2726,8 @@ function exportResourceLocks { } function getConsumption { + $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) + function addToAllConsumptionData { [CmdletBinding()]Param( [Parameter(Mandatory)] @@ -2738,7 +2762,7 @@ function getConsumption { $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions (QuotaId Whitelist: '$($SubscriptionQuotaIdWhitelist -join ', ')') for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $counterBatch = [PSCustomObject] @{ Value = 0 } @@ -2885,13 +2909,14 @@ function getConsumption { $htConsumptionExceptionLog = $using:htConsumptionExceptionLog #other $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)' ($($subscriptionQuotaIdToProcess)) (whitelist))" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { @@ -2954,7 +2979,7 @@ function getConsumption { $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $body = @" { @@ -3080,13 +3105,14 @@ function getConsumption { $htConsumptionExceptionLog = $using:htConsumptionExceptionLog #other $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)' ($($subscriptionQuotaIdToProcess)))" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { @@ -3807,7 +3833,7 @@ resources | where type =~ 'microsoft.network/publicIpAddresses' resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' -| where not(name endswith "-asr") +| where not(name endswith '-asr') | project type, subscriptionId, Resource=id, Intent='$intent' "@ intent = $intent @@ -29928,7 +29954,8 @@ function dataCollectionDefenderPlans { $currentTask = "Getting Microsoft Defender for Cloud plans for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$SubscriptionQuotaId']" #https://learn.microsoft.com/rest/api/defenderforcloud/pricings - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=2024-01-01" + $securitypricingsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.securityPricings.($azAPICallConf['htParameters'].azureCloudEnvironment) + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=$($securitypricingsAPIVersion)" $method = 'GET' $defenderPlansResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' @@ -33351,13 +33378,15 @@ function dataCollectionRoleDefinitions { $subscriptionQuotaId ) + $roledefinitionsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.roledefinitions.($azAPICallConf['htParameters'].azureCloudEnvironment) + if ($TargetMgOrSub -eq 'Sub') { $currentTask = "Getting Custom Role definitions for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'CustomRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'CustomRole'" } if ($TargetMgOrSub -eq 'MG') { $currentTask = "Getting Custom Role definitions for Management Group: '$($scopeDisplayName)' ('$scopeId')" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'CustomRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'CustomRole'" } $method = 'GET' $scopeCustomRoleDefinitions = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 478e8bf5..1382efbc 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.6', + $ProductVersion = '6.4.7', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -627,7 +627,26 @@ Param $MSTenantIds = @('2f4a9838-26b7-47ee-be60-ccc1fdec5953', '33e01921-4d64-4f8c-a055-5bdaffd5e33d'), [array] - $ValidPolicyEffects = @('addToNetworkGroup', 'append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting', 'mutate') + $ValidPolicyEffects = @('addToNetworkGroup', 'append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting', 'mutate'), + + [hashtable] + $APIMappingCloudEnvironment = @{ + roleDefinitions = @{ + AzureCloud = '2023-07-01-preview' + AzureUSGovernment = '2022-05-01-preview' + AzureChinaCloud = '2022-05-01-preview' + } + costManagementQuery = @{ + AzureCloud = '2024-01-01' + AzureUSGovernment = '2023-09-01' + AzureChinaCloud = '2023-09-01' + } + securityPricings = @{ + AzureCloud = '2024-01-01' + AzureUSGovernment = '2023-01-01' + AzureChinaCloud = '2023-01-01' + } + } ) $Error.clear() diff --git a/pwsh/dev/functions/addHtParameters.ps1 b/pwsh/dev/functions/addHtParameters.ps1 index 9e6d413a..cca20a1f 100644 --- a/pwsh/dev/functions/addHtParameters.ps1 +++ b/pwsh/dev/functions/addHtParameters.ps1 @@ -40,6 +40,7 @@ GitHubActionsOIDC = [bool]$GitHubActionsOIDC NoNetwork = [bool]$NoNetwork ThrottleLimit = $ThrottleLimit + APIMappingCloudEnvironment = $APIMappingCloudEnvironment } Write-Host 'htParameters:' $azAPICallConf['htParameters'] | Format-Table -AutoSize | Out-String diff --git a/pwsh/dev/functions/cacheBuiltIn.ps1 b/pwsh/dev/functions/cacheBuiltIn.ps1 index 0c82101f..5a07ed5c 100644 --- a/pwsh/dev/functions/cacheBuiltIn.ps1 +++ b/pwsh/dev/functions/cacheBuiltIn.ps1 @@ -220,16 +220,18 @@ if ($builtInCapability -eq 'RoleDefinitions') { + $roledefinitionsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.roledefinitions.($azAPICallConf['htParameters'].azureCloudEnvironment) + #region subscriptionScope if ($ignoreARMLocation) { $currentTask = 'Caching built-in Role definitions (subscriptionScope)' Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } else { $currentTask = "Caching built-in Role definitions (Location: '$($ARMLocation)') (subscriptionScope)" Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/subscriptions/$($azAPICallConf['checkContext'].Subscription.Id)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } $method = 'GET' @@ -281,12 +283,12 @@ if ($ignoreARMLocation) { $currentTask = 'Caching built-in Role definitions (tenantScope)' Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].'ARM')/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } else { $currentTask = "Caching built-in Role definitions (Location: '$($ARMLocation)') (tenantScope)" Write-Host " $currentTask" - $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'BuiltInRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls']."ARM$($ARMLocation)")/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'BuiltInRole'" } $method = 'GET' diff --git a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 index 60f341b5..f54d2dfa 100644 --- a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 +++ b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 @@ -28,7 +28,8 @@ function dataCollectionDefenderPlans { $currentTask = "Getting Microsoft Defender for Cloud plans for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$SubscriptionQuotaId']" #https://learn.microsoft.com/rest/api/defenderforcloud/pricings - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=2024-01-01" + $securitypricingsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.securityPricings.($azAPICallConf['htParameters'].azureCloudEnvironment) + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=$($securitypricingsAPIVersion)" $method = 'GET' $defenderPlansResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' @@ -3451,13 +3452,15 @@ function dataCollectionRoleDefinitions { $subscriptionQuotaId ) + $roledefinitionsAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.roledefinitions.($azAPICallConf['htParameters'].azureCloudEnvironment) + if ($TargetMgOrSub -eq 'Sub') { $currentTask = "Getting Custom Role definitions for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'CustomRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'CustomRole'" } if ($TargetMgOrSub -eq 'MG') { $currentTask = "Getting Custom Role definitions for Management Group: '$($scopeDisplayName)' ('$scopeId')" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=2023-07-01-preview&`$filter=type eq 'CustomRole'" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($scopeId)/providers/Microsoft.Authorization/roleDefinitions?api-version=$($roledefinitionsAPIVersion)&`$filter=type eq 'CustomRole'" } $method = 'GET' $scopeCustomRoleDefinitions = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' diff --git a/pwsh/dev/functions/getConsumption.ps1 b/pwsh/dev/functions/getConsumption.ps1 index 885962fd..78433ac6 100644 --- a/pwsh/dev/functions/getConsumption.ps1 +++ b/pwsh/dev/functions/getConsumption.ps1 @@ -1,5 +1,7 @@ function getConsumption { + $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) + function addToAllConsumptionData { [CmdletBinding()]Param( [Parameter(Mandatory)] @@ -34,7 +36,7 @@ $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions (QuotaId Whitelist: '$($SubscriptionQuotaIdWhitelist -join ', ')') for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $counterBatch = [PSCustomObject] @{ Value = 0 } @@ -181,13 +183,14 @@ $htConsumptionExceptionLog = $using:htConsumptionExceptionLog #other $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)' ($($subscriptionQuotaIdToProcess)) (whitelist))" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { @@ -250,7 +253,7 @@ $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $body = @" { @@ -376,13 +379,14 @@ $htConsumptionExceptionLog = $using:htConsumptionExceptionLog #other $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)' ($($subscriptionQuotaIdToProcess)))" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=2024-01-01&`$top=5000" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $method = 'POST' $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { diff --git a/pwsh/dev/functions/getOrphanedResources.ps1 b/pwsh/dev/functions/getOrphanedResources.ps1 index bb928eee..a932a120 100644 --- a/pwsh/dev/functions/getOrphanedResources.ps1 +++ b/pwsh/dev/functions/getOrphanedResources.ps1 @@ -114,7 +114,7 @@ resources | where type =~ 'microsoft.network/publicIpAddresses' resources | where type =~ 'microsoft.compute/availabilitySets' | where properties.virtualMachines == '[]' -| where not(name endswith "-asr") +| where not(name endswith '-asr') | project type, subscriptionId, Resource=id, Intent='$intent' "@ intent = $intent diff --git a/version.json b/version.json index 13ebe353..c80cd715 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.6" + "ProductVersion": "6.4.7" } \ No newline at end of file From f4b4f1ee6fc79359abbaaeb3a6556bd38aa7de1c Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 23 May 2024 14:18:33 +0200 Subject: [PATCH 10/20] 6.4.7 --- README.md | 3 ++- history.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 851cc7da..91cafd74 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,11 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-May-15 / 6.4.6 Minor) +**Changes** (2024-May-23 / 6.4.7 Minor) - DevSkim and PSScriptAnalyzer integration - fixes and optimization based on DevSkim and PSScriptAnalyzer findings +- api version mapping in param for cloud environment api version availability drift [Full release history](history.md) diff --git a/history.md b/history.md index 3bfd792a..85711aed 100644 --- a/history.md +++ b/history.md @@ -4,10 +4,11 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-May-15 / 6.4.6 Minor) +**Changes** (2024-May-23 / 6.4.7 Minor) - DevSkim and PSScriptAnalyzer integration - fixes and optimization based on DevSkim and PSScriptAnalyzer findings +- api version mapping in param for cloud environment api version availability drift **Changes** (2024-May-05 / 6.4.5 Minor) From 9d81203a3aed021c683c7da1fe92edd1504bd570 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Fri, 24 May 2024 12:58:13 +0200 Subject: [PATCH 11/20] Scorecard supply-chain security --- .github/workflows/scorecard.yml | 74 +++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/scorecard.yml diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..61b8d1b7 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,74 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + # schedule: + # - cron: '21 0 * * 1' + push: + branches: [ "developmentJH" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + if: github.repository == 'JulianHayward/Azure-MG-Sub-Governance-Reporting' + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 + with: + sarif_file: results.sarif \ No newline at end of file From 347994c3e29208604d9b46a22f442ee0aee2ee0a Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 27 May 2024 18:08:16 +0200 Subject: [PATCH 12/20] 6.4.7 --- README.md | 8 ++++---- history.md | 8 ++++---- pwsh/AzGovVizParallel.ps1 | 2 +- pwsh/dev/devAzGovVizParallel.ps1 | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 91cafd74..ac6d097f 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,11 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-May-23 / 6.4.7 Minor) +**Changes** (2024-May-24 / 6.4.7 Minor) -- DevSkim and PSScriptAnalyzer integration -- fixes and optimization based on DevSkim and PSScriptAnalyzer findings -- api version mapping in param for cloud environment api version availability drift +- [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration +- fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings +- api version mapping in param block for cloud environment api version availability drift [Full release history](history.md) diff --git a/history.md b/history.md index 85711aed..7ddf5ace 100644 --- a/history.md +++ b/history.md @@ -4,11 +4,11 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-May-23 / 6.4.7 Minor) +**Changes** (2024-May-24 / 6.4.7 Minor) -- DevSkim and PSScriptAnalyzer integration -- fixes and optimization based on DevSkim and PSScriptAnalyzer findings -- api version mapping in param for cloud environment api version availability drift +- [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration +- fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings +- api version mapping in param block for cloud environment api version availability drift **Changes** (2024-May-05 / 6.4.5 Minor) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 5f707e83..3a581957 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -388,7 +388,7 @@ Param # AzAPICall related parameters ---> [string] - $ARMLocation = 'northeurope', + $ARMLocation = 'westeurope', [string] $ScriptPath = 'pwsh', #e.g. 'myfolder\pwsh' diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 1382efbc..531be3d6 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -388,7 +388,7 @@ Param # AzAPICall related parameters ---> [string] - $ARMLocation = 'northeurope', + $ARMLocation = 'westeurope', [string] $ScriptPath = 'pwsh', #e.g. 'myfolder\pwsh' From 5b153d911126bba94b72fb31ec028703462e5ed0 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 27 May 2024 18:47:27 +0200 Subject: [PATCH 13/20] 6.4.7 --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index ac6d097f..a35fe28d 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ Azure Governance Visualizer is intended to help you to get a holistic overview o - [Azure Governance Visualizer @ Microsoft CAF](#azure-governance-visualizer--microsoft-caf) - [Microsoft Cloud Adoption Framework (CAF)](#microsoft-cloud-adoption-framework-caf) - [Azure Governance Visualizer accelerator](#azure-governance-visualizer-accelerator) - - [ChatGPT](#chatgpt) - [:rocket: Azure Governance Visualizer deployment guide](#rocket-azure-governance-visualizer-deployment-guide) - [Release history](#release-history) - [Demo](#demo) @@ -75,10 +74,6 @@ Azure Governance Visualizer is intended to help you to get a holistic overview o The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Governance-Visualizer-Accelerator) provides an easy and fast deployment process that automates the creation and publishing of AzGovViz to an Azure Web Application and provides automation to configuring the pre-requisites for AzGovViz. -## ChatGPT - -![ChatGPT](img/chatGPT.png) - ## :rocket: Azure Governance Visualizer deployment guide The instructions to deploy the Azure Governance Visualizer is found in the **[Azure Governance Visualizer (AzGovViz) deployment guide](setup.md)**. Follow those instructions to run AzGovViz from your terminal (console), GitHub Codepaces, Azure DevOps, or GitHub. From 19e96c8c1703980807115f2e6cd6e93fdfec5522 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 3 Jun 2024 16:49:04 +0200 Subject: [PATCH 14/20] 6.4.8 --- README.md | 3 +- history.md | 3 +- pwsh/AzGovVizParallel.ps1 | 116 +++++++++++++++++- pwsh/dev/devAzGovVizParallel.ps1 | 2 +- .../processALZPolicyVersionChecker.ps1 | 114 +++++++++++++++++ version.json | 2 +- 6 files changed, 235 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a35fe28d..aa8de273 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,9 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-May-24 / 6.4.7 Minor) +**Changes** (2024-June-03 / 6.4.8 Minor) +- ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration - fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings - api version mapping in param block for cloud environment api version availability drift diff --git a/history.md b/history.md index 7ddf5ace..68de6e8e 100644 --- a/history.md +++ b/history.md @@ -4,8 +4,9 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-May-24 / 6.4.7 Minor) +**Changes** (2024-June-03 / 6.4.8 Minor) +- ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration - fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings - api version mapping in param block for cloud environment api version availability drift diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 3a581957..b14abae7 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.7', + $ProductVersion = '6.4.8', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -5489,6 +5489,120 @@ function processALZPolicyVersionChecker { } } + #ALZ policy refresh H2 FY24 (initiatives.json) + $gitHistInitiatives = (git log --format="%ai`t%H`t%an`t%ae`t%s" -- ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json) | ConvertFrom-Csv -Delimiter "`t" -Header ('Date', 'CommitId', 'Author', 'Email', 'Subject') + $commitCount = 0 + #$doNewALZPolicyReadingApproach = $false + foreach ($commit in $gitHistInitiatives | Sort-Object -Property Date) { + + # if ($commit.CommitId -eq $ALZCommitId) { + # $doNewALZPolicyReadingApproach = $true + # } + #Write-Host "processing commit $($commit.CommitId) - doNewALZPolicyReadingApproach: $doNewALZPolicyReadingApproach" + $commitCount++ + + $jsonRaw = git show "$($commit.CommitId):eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json" + + #if ($doNewALZPolicyReadingApproach) { + $jsonESLZPolicySets = $jsonRaw -replace '\[\[', '[' | ConvertFrom-Json + [regex]$extractVariableName = "(?<=\[variables\(')[^']+" + + $refsPolicySetDefinitionsAll = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.All).Value + $refsPolicySetDefinitionsAzureCloud = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureCloud).Value + $refsPolicySetDefinitionsAzureChinaCloud = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureChinaCloud).Value + $refsPolicySetDefinitionsAzureUSGovernment = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureUSGovernment).Value + $listPolicySetDefinitionsAzureCloud = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureCloud + $listPolicySetDefinitionsAzureChinaCloud = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureChinaCloud + $listPolicySetDefinitionsAzureUSGovernment = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureUSGovernment + $policySetDefinitionsAzureCloud = $listPolicySetDefinitionsAzureCloud.ForEach({ $jsonESLZPolicySets.variables.$_ }) + $policySetDefinitionsAzureChinaCloud = $listPolicySetDefinitionsAzureChinaCloud.ForEach({ $jsonESLZPolicySets.variables.$_ }) + $policySetDefinitionsAzureUSGovernment = $listPolicySetDefinitionsAzureUSGovernment.ForEach({ $jsonESLZPolicySets.variables.$_ }) + + switch ($azAPICallConf['checkContext'].Environment.Name) { + 'Azurecloud' { + $policySetDefinitionsData = $policySetDefinitionsAzureCloud + } + 'AzureChinaCloud' { + $policySetDefinitionsData = $policySetDefinitionsAzureChinaCloud + } + 'AzureUSGovernment' { + $policySetDefinitionsData = $policySetDefinitionsAzureUSGovernment + } + } + + foreach ($policySetDefinition in $policySetDefinitionsData) { + + $policyJsonRebuild = $policySetDefinition | ConvertFrom-Json + $policyJsonParameters = $policyJsonRebuild.properties.parameters | ConvertTo-Json -Depth 99 + $policyJsonPolicyDefinitions = $policyJsonRebuild.properties.policyDefinitions | ConvertTo-Json -Depth 99 + $hashParameters = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonParameters)) + $stringHashParameters = [System.BitConverter]::ToString($hashParameters) + $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) + $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) + $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + + if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { + $allESLZPolicySets.($policyJsonRebuild.name) = @{} + $allESLZPolicySets.($policyJsonRebuild.name).version = [System.Collections.ArrayList]@() + $null = $allESLZPolicySets.($policyJsonRebuild.name).version.Add($policyJsonRebuild.properties.metadata.version) + $allESLZPolicySets.($policyJsonRebuild.name).$stringHash = $policyJsonRebuild.properties.metadata.version + $allESLZPolicySets.($policyJsonRebuild.name).name = $policyJsonRebuild.name + $allESLZPolicySets.($policyJsonRebuild.name).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'prod' + } + else { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'obsolete' + } + } + else { + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'prod' + } + else { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'obsolete' + } + $allESLZPolicySets.($policyJsonRebuild.name).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($allESLZPolicySets.($policyJsonRebuild.name).version -notcontains $policyJsonRebuild.properties.metadata.version) { + $null = $allESLZPolicySets.($policyJsonRebuild.name).version.Add($policyJsonRebuild.properties.metadata.version) + } + if (-not $allESLZPolicySets.($policyJsonRebuild.name).$stringHash) { + $allESLZPolicySets.($policyJsonRebuild.name).$stringHash = $policyJsonRebuild.properties.metadata.version + } + } + + #hsh + if (-not $allESLZPolicySetHashes.($stringHash)) { + $allESLZPolicySetHashes.($stringHash) = @{} + $allESLZPolicySetHashes.($stringHash).version = [System.Collections.ArrayList]@() + $null = $allESLZPolicySetHashes.($stringHash).version.Add($policyJsonRebuild.properties.metadata.version) + $allESLZPolicySetHashes.($stringHash).name = $policyJsonRebuild.name + $allESLZPolicySetHashes.($stringHash).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySetHashes.($stringHash).status = 'prod' + } + else { + $allESLZPolicySetHashes.($stringHash).status = 'obsolete' + } + } + else { + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySetHashes.($stringHash).status = 'prod' + } + else { + $allESLZPolicySetHashes.($stringHash).status = 'obsolete' + } + $allESLZPolicySetHashes.($stringHash).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($allESLZPolicySetHashes.($stringHash).version -notcontains $policyJsonRebuild.properties.metadata.version) { + $null = $allESLZPolicySetHashes.($stringHash).version.Add($policyJsonRebuild.properties.metadata.version) + } + if (-not $allESLZPolicySetHashes.($stringHash).($policyJsonRebuild.name)) { + $allESLZPolicySetHashes.($stringHash).($policyJsonRebuild.name) = $policyJsonRebuild.name + } + } + } + #} + } Write-Host " $($allESLZPolicies.Keys.Count) Azure Landing Zones (ALZ) Policy definitions ($($allESLZPolicies.Values.where({$_.status -eq 'Prod'}).Count) productive)" Write-Host " $($allESLZPolicySets.Keys.Count) Azure Landing Zones (ALZ) PolicySet definitions ($($allESLZPolicySets.Values.where({$_.status -eq 'Prod'}).Count) productive)" diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 531be3d6..67c10156 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.7', + $ProductVersion = '6.4.8', [string] $GithubRepository = 'aka.ms/AzGovViz', diff --git a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 index 9763b691..12424869 100644 --- a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 +++ b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 @@ -473,6 +473,120 @@ } } + #ALZ policy refresh H2 FY24 (initiatives.json) + $gitHistInitiatives = (git log --format="%ai`t%H`t%an`t%ae`t%s" -- ./eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json) | ConvertFrom-Csv -Delimiter "`t" -Header ('Date', 'CommitId', 'Author', 'Email', 'Subject') + $commitCount = 0 + #$doNewALZPolicyReadingApproach = $false + foreach ($commit in $gitHistInitiatives | Sort-Object -Property Date) { + + # if ($commit.CommitId -eq $ALZCommitId) { + # $doNewALZPolicyReadingApproach = $true + # } + #Write-Host "processing commit $($commit.CommitId) - doNewALZPolicyReadingApproach: $doNewALZPolicyReadingApproach" + $commitCount++ + + $jsonRaw = git show "$($commit.CommitId):eslzArm/managementGroupTemplates/policyDefinitions/initiatives.json" + + #if ($doNewALZPolicyReadingApproach) { + $jsonESLZPolicySets = $jsonRaw -replace '\[\[', '[' | ConvertFrom-Json + [regex]$extractVariableName = "(?<=\[variables\(')[^']+" + + $refsPolicySetDefinitionsAll = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.All).Value + $refsPolicySetDefinitionsAzureCloud = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureCloud).Value + $refsPolicySetDefinitionsAzureChinaCloud = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureChinaCloud).Value + $refsPolicySetDefinitionsAzureUSGovernment = $extractVariableName.Matches($jsonESLZPolicySets.variables.loadPolicySetDefinitions.AzureUSGovernment).Value + $listPolicySetDefinitionsAzureCloud = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureCloud + $listPolicySetDefinitionsAzureChinaCloud = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureChinaCloud + $listPolicySetDefinitionsAzureUSGovernment = $refsPolicySetDefinitionsAll + $refsPolicySetDefinitionsAzureUSGovernment + $policySetDefinitionsAzureCloud = $listPolicySetDefinitionsAzureCloud.ForEach({ $jsonESLZPolicySets.variables.$_ }) + $policySetDefinitionsAzureChinaCloud = $listPolicySetDefinitionsAzureChinaCloud.ForEach({ $jsonESLZPolicySets.variables.$_ }) + $policySetDefinitionsAzureUSGovernment = $listPolicySetDefinitionsAzureUSGovernment.ForEach({ $jsonESLZPolicySets.variables.$_ }) + + switch ($azAPICallConf['checkContext'].Environment.Name) { + 'Azurecloud' { + $policySetDefinitionsData = $policySetDefinitionsAzureCloud + } + 'AzureChinaCloud' { + $policySetDefinitionsData = $policySetDefinitionsAzureChinaCloud + } + 'AzureUSGovernment' { + $policySetDefinitionsData = $policySetDefinitionsAzureUSGovernment + } + } + + foreach ($policySetDefinition in $policySetDefinitionsData) { + + $policyJsonRebuild = $policySetDefinition | ConvertFrom-Json + $policyJsonParameters = $policyJsonRebuild.properties.parameters | ConvertTo-Json -Depth 99 + $policyJsonPolicyDefinitions = $policyJsonRebuild.properties.policyDefinitions | ConvertTo-Json -Depth 99 + $hashParameters = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonParameters)) + $stringHashParameters = [System.BitConverter]::ToString($hashParameters) + $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) + $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) + $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + + if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { + $allESLZPolicySets.($policyJsonRebuild.name) = @{} + $allESLZPolicySets.($policyJsonRebuild.name).version = [System.Collections.ArrayList]@() + $null = $allESLZPolicySets.($policyJsonRebuild.name).version.Add($policyJsonRebuild.properties.metadata.version) + $allESLZPolicySets.($policyJsonRebuild.name).$stringHash = $policyJsonRebuild.properties.metadata.version + $allESLZPolicySets.($policyJsonRebuild.name).name = $policyJsonRebuild.name + $allESLZPolicySets.($policyJsonRebuild.name).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'prod' + } + else { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'obsolete' + } + } + else { + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'prod' + } + else { + $allESLZPolicySets.($policyJsonRebuild.name).status = 'obsolete' + } + $allESLZPolicySets.($policyJsonRebuild.name).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($allESLZPolicySets.($policyJsonRebuild.name).version -notcontains $policyJsonRebuild.properties.metadata.version) { + $null = $allESLZPolicySets.($policyJsonRebuild.name).version.Add($policyJsonRebuild.properties.metadata.version) + } + if (-not $allESLZPolicySets.($policyJsonRebuild.name).$stringHash) { + $allESLZPolicySets.($policyJsonRebuild.name).$stringHash = $policyJsonRebuild.properties.metadata.version + } + } + + #hsh + if (-not $allESLZPolicySetHashes.($stringHash)) { + $allESLZPolicySetHashes.($stringHash) = @{} + $allESLZPolicySetHashes.($stringHash).version = [System.Collections.ArrayList]@() + $null = $allESLZPolicySetHashes.($stringHash).version.Add($policyJsonRebuild.properties.metadata.version) + $allESLZPolicySetHashes.($stringHash).name = $policyJsonRebuild.name + $allESLZPolicySetHashes.($stringHash).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySetHashes.($stringHash).status = 'prod' + } + else { + $allESLZPolicySetHashes.($stringHash).status = 'obsolete' + } + } + else { + if ($commitCount -eq $gitHistInitiatives.Count) { + $allESLZPolicySetHashes.($stringHash).status = 'prod' + } + else { + $allESLZPolicySetHashes.($stringHash).status = 'obsolete' + } + $allESLZPolicySetHashes.($stringHash).metadataSource = $policyJsonRebuild.properties.metadata.source + if ($allESLZPolicySetHashes.($stringHash).version -notcontains $policyJsonRebuild.properties.metadata.version) { + $null = $allESLZPolicySetHashes.($stringHash).version.Add($policyJsonRebuild.properties.metadata.version) + } + if (-not $allESLZPolicySetHashes.($stringHash).($policyJsonRebuild.name)) { + $allESLZPolicySetHashes.($stringHash).($policyJsonRebuild.name) = $policyJsonRebuild.name + } + } + } + #} + } Write-Host " $($allESLZPolicies.Keys.Count) Azure Landing Zones (ALZ) Policy definitions ($($allESLZPolicies.Values.where({$_.status -eq 'Prod'}).Count) productive)" Write-Host " $($allESLZPolicySets.Keys.Count) Azure Landing Zones (ALZ) PolicySet definitions ($($allESLZPolicySets.Values.where({$_.status -eq 'Prod'}).Count) productive)" diff --git a/version.json b/version.json index c80cd715..b642bbbb 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.7" + "ProductVersion": "6.4.8" } \ No newline at end of file From ce9c47bd75d0495ff727d2e74be40a38dadaa5ec Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Tue, 18 Jun 2024 17:59:08 +0200 Subject: [PATCH 15/20] 6.4.9 --- .github/workflows/AzGovViz.yml | 2 +- .github/workflows/AzGovViz_OIDC.yml | 4 +- README.md | 6 +- history.md | 6 +- pwsh/AzGovVizParallel.ps1 | 423 +++++++++++++++++- pwsh/dev/devAzGovVizParallel.ps1 | 3 +- pwsh/dev/functions/getConsumptionv2.ps1 | 418 +++++++++++++++++ .../processALZPolicyVersionChecker.ps1 | 3 + version.json | 2 +- 9 files changed, 859 insertions(+), 8 deletions(-) create mode 100644 pwsh/dev/functions/getConsumptionv2.ps1 diff --git a/.github/workflows/AzGovViz.yml b/.github/workflows/AzGovViz.yml index ae91cef6..f20bb2ed 100644 --- a/.github/workflows/AzGovViz.yml +++ b/.github/workflows/AzGovViz.yml @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@v2 - name: Connect Azure - uses: azure/login@v1 + uses: azure/login@v2 with: creds: ${{secrets.CREDS}} enable-AzPSSession: true diff --git a/.github/workflows/AzGovViz_OIDC.yml b/.github/workflows/AzGovViz_OIDC.yml index 40544585..76cfdb5d 100644 --- a/.github/workflows/AzGovViz_OIDC.yml +++ b/.github/workflows/AzGovViz_OIDC.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@v2 - name: Connect Azure OIDC - uses: azure/login@v1 + uses: azure/login@v2 with: client-id: ${{secrets.CLIENT_ID}} #create this secret tenant-id: ${{secrets.TENANT_ID}} #create this secret @@ -91,7 +91,7 @@ jobs: #log again to avoid timeout before web publishing - name: Connect Azure OIDC if: env.WebAppPublish == 'true' - uses: azure/login@v1 + uses: azure/login@v2 with: client-id: ${{secrets.CLIENT_ID}} #create this secret (GitHub/Setting/Secrets) tenant-id: ${{secrets.TENANT_ID}} #create this secret diff --git a/README.md b/README.md index aa8de273..68dd29cc 100644 --- a/README.md +++ b/README.md @@ -82,12 +82,16 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-June-03 / 6.4.8 Minor) +**Changes** (2024-June-18 / 6.4.9 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration - fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings - api version mapping in param block for cloud environment api version availability drift +- update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): + - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) + - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) +- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` [Full release history](history.md) diff --git a/history.md b/history.md index 68de6e8e..1481e6f9 100644 --- a/history.md +++ b/history.md @@ -4,12 +4,16 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-June-03 / 6.4.8 Minor) +**Changes** (2024-June-18 / 6.4.9 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration - fixes and optimization based on DevSkim, PSScriptAnalyzer and OpenSSF Scorecard findings - api version mapping in param block for cloud environment api version availability drift +- update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): + - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) + - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) +- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` **Changes** (2024-May-05 / 6.4.5 Minor) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index b14abae7..30e60a91 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.8', + $ProductVersion = '6.4.9', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -3335,6 +3335,424 @@ function getConsumption { $endConsumptionData = Get-Date Write-Host "Getting Consumption data duration: $((New-TimeSpan -Start $startConsumptionData -End $endConsumptionData).TotalSeconds) seconds" } +function getConsumptionv2 { + #todo: remove + Write-Host '#########-----------------#########' -ForegroundColor DarkMagenta + Write-Host 'Executing getConsumptionv2' -ForegroundColor DarkMagenta + + $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) + + function addToAllConsumptionData { + [CmdletBinding()]Param( + [Parameter(Mandatory)] + [object] + $consumptiondataFromAPI, + + [Parameter(Mandatory)] + [string] + $subscriptionQuotaId + ) + + foreach ($consumptionline in $consumptiondataFromAPI.properties.rows) { + $hlper = $htSubscriptionsMgPath.($consumptionline[1]) + + $null = $script:allConsumptionData.Add([PSCustomObject]@{ + "$($consumptiondataFromAPI.properties.columns.name[0])" = [decimal]$consumptionline[0] + "$($consumptiondataFromAPI.properties.columns.name[1])" = $consumptionline[1] + SubscriptionName = $hlper.DisplayName + subscriptionQuotaId = $subscriptionQuotaId + SubscriptionMgPath = $hlper.ParentNameChainDelimited + "$($consumptiondataFromAPI.properties.columns.name[2])" = $consumptionline[2] + "$($consumptiondataFromAPI.properties.columns.name[3])" = $consumptionline[3] + "$($consumptiondataFromAPI.properties.columns.name[4])" = $consumptionline[4] + "$($consumptiondataFromAPI.properties.columns.name[5])" = $consumptionline[5] + "$($consumptiondataFromAPI.properties.columns.name[6])" = $consumptionline[6] + }) + } + } + + $startConsumptionData = Get-Date + + if ($subsToProcessInCustomDataCollectionCount -gt 0) { + + #$subscriptionIdsOptimizedForBody = '"{0}"' -f ($subsToProcessInCustomDataCollection.subscriptionId -join '","') + $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" + #https://learn.microsoft.com/rest/api/cost-management/query/usage + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=100" + $method = 'POST' + + $subsToProcessInCustomDataCollectionGroupedByQuotaId = $subsToProcessInCustomDataCollection | Group-Object -Property subscriptionQuotaId + $cnter = 0 + foreach ($quotaIdGroup in $subsToProcessInCustomDataCollectionGroupedByQuotaId) { + + $counterBatch = [PSCustomObject] @{ Value = 0 } + $batchSize = 100 + $subscriptionsBatch = ($quotaIdGroup.Group) | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } + $batchCnt = 0 + Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId $($quotaIdGroup.Name) in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" + + foreach ($batch in $subscriptionsBatch) { + $cnter++ + $batchCnt++ + $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') + $currenttask = " Getting Consumption data quotaId:'$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" -ForegroundColor Cyan + + $body = @" + { + "type": "ActualCost", + "dataset": { + "granularity": "none", + "filter": { + "dimensions": { + "name": "SubscriptionId", + "operator": "In", + "values": [ + $($subscriptionIdsOptimizedForBody) + ] + } + }, + "aggregation": { + "totalCost": { + "name": "PreTaxCost", + "function": "Sum" + } + }, + "grouping": [ + { + "type": "Dimension", + "name": "SubscriptionId" + }, + { + "type": "Dimension", + "name": "ResourceId" + }, + { + "type": "Dimension", + "name": "ResourceType" + }, + { + "type": "Dimension", + "name": "MeterCategory" + }, + { + "type": "Dimension", + "name": "ChargeType" + } + ] + }, + "timeframe": "Custom", + "timeperiod": { + "from": "$($azureConsumptionStartDate)", + "to": "$($azureConsumptionEndDate)" + } + } +"@ + + $mgConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + + <#test + #$mgConsumptionData = "OfferNotSupported" + if ($batchCnt -eq 1){ + $mgConsumptionData = "OfferNotSupported" + } + #> + #enforce switch to 'foreach Subscription' Subscription scope mode + # if ($cnter -eq 2) { + # $mgConsumptionData = 'Unauthorized' + # } + if ($mgConsumptionData -eq 'Unauthorized' -or $mgConsumptionData -eq 'OfferNotSupported' -or $mgConsumptionData -eq 'NoValidSubscriptions') { + if (-not $script:htConsumptionExceptionLog.Mg.($ManagementGroupId)) { + $script:htConsumptionExceptionLog.Mg.($ManagementGroupId) = @{} + } + $script:htConsumptionExceptionLog.Mg.($ManagementGroupId).($batchCnt) = @{ + Exception = $mgConsumptionData + Subscriptions = ($batch.Group).subscriptionId + } + + Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data #batch$($batchCnt) using Management Group scope failed." + $body = @" + { + "type": "ActualCost", + "dataset": { + "granularity": "none", + "aggregation": { + "totalCost": { + "name": "PreTaxCost", + "function": "Sum" + } + }, + "grouping": [ + { + "type": "Dimension", + "name": "SubscriptionId" + }, + { + "type": "Dimension", + "name": "ResourceId" + }, + { + "type": "Dimension", + "name": "ResourceType" + }, + { + "type": "Dimension", + "name": "MeterCategory" + }, + { + "type": "Dimension", + "name": "ChargeType" + } + ] + }, + "timeframe": "Custom", + "timeperiod": { + "from": "$($azureConsumptionStartDate)", + "to": "$($azureConsumptionEndDate)" + } + } +"@ + $funcAddToAllConsumptionData = $function:addToAllConsumptionData.ToString() + $batch.Group | ForEach-Object -Parallel { + $subIdToProcess = $_.subscriptionId + $subNameToProcess = $_.subscriptionName + $subQuotaId = $_.subscriptionQuotaId + #region UsingVARs + $body = $using:body + $azureConsumptionStartDate = $using:azureConsumptionStartDate + $azureConsumptionEndDate = $using:azureConsumptionEndDate + #fromOtherFunctions + $azAPICallConf = $using:azAPICallConf + $scriptPath = $using:ScriptPath + #Array&HTs + $allConsumptionData = $using:allConsumptionData + $htSubscriptionsMgPath = $using:htSubscriptionsMgPath + $htAllSubscriptionsFromAPI = $using:htAllSubscriptionsFromAPI + $htConsumptionExceptionLog = $using:htConsumptionExceptionLog + #other + $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion + #endregion UsingVARs + + $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + #test + Write-Host $currentTask + #https://learn.microsoft.com/rest/api/cost-management/query/usage + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" + $method = 'POST' + $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { + Write-Host " Failed ($subConsumptionData) - Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails + $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) + $script:htConsumptionExceptionLog.Sub.($subIdToProcess) = @{ + Exception = $subConsumptionData + SubscriptionId = $subIdToProcess + SubscriptionName = $hlper.displayName + QuotaId = $hlper.subscriptionPolicies.quotaId + mgPath = $hlper2.ParentNameChainDelimited + mgParent = $hlper2.Parent + } + + Continue + } + else { + Write-Host " $($subConsumptionData.properties.rows.Count) Consumption data entries (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + if ($subConsumptionData.Count -gt 0) { + addToAllConsumptionData -consumptiondataFromAPI $subConsumptionData -subscriptionQuotaId $subQuotaId + } + } + } -ThrottleLimit $ThrottleLimit + } + else { + Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" + if ($mgConsumptionData.Count -gt 0) { + addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name + } + } + } + } + + } + else { + $detailShowStopperResult = 'NoSubscriptionsPresent' + Write-Host ' No Subscriptions present, skipping Consumption data processing' + } + + + if ($detailShowStopperResult -eq 'AccountCostDisabled' -or $detailShowStopperResult -eq 'NoValidSubscriptions' -or $detailShowStopperResult -eq 'NoSubscriptionsPresent') { + if ($detailShowStopperResult -eq 'AccountCostDisabled') { + Write-Host ' Seems Access to cost data has been disabled for this Account - skipping CostManagement' + } + if ($detailShowStopperResult -eq 'NoValidSubscriptions') { + Write-Host ' Seems there are no valid Subscriptions present - skipping CostManagement' + } + if ($detailShowStopperResult -eq 'NoSubscriptionsPresent') { + Write-Host ' Seems there are no Subscriptions present - skipping CostManagement' + } + Write-Host " Action: Setting switch parameter 'DoAzureConsumption' to false" + $azAPICallConf['htParameters'].DoAzureConsumption = $false + } + else { + Write-Host ' Checking returned Consumption data' + $script:allConsumptionDataCount = $allConsumptionData.Count + + if ($allConsumptionDataCount -gt 0) { + + $script:allConsumptionData = $allConsumptionData.where( { $_.PreTaxCost -ne 0 } ) + $script:allConsumptionDataCount = $allConsumptionData.Count + + if ($allConsumptionDataCount -gt 0) { + Write-Host " $($allConsumptionDataCount) relevant Consumption data entries" + + $script:consumptionData = $allConsumptionData + $script:consumptionDataGroupedByCurrency = $consumptionData | Group-Object -Property Currency + + foreach ($currency in $consumptionDataGroupedByCurrency) { + + #subscriptions + $groupAllConsumptionDataPerCurrencyBySubscriptionId = $currency.group | Group-Object -Property SubscriptionId + foreach ($subscriptionId in $groupAllConsumptionDataPerCurrencyBySubscriptionId) { + + $subTotalCost = ($subscriptionId.Group.PreTaxCost | Measure-Object -Sum).Sum + $script:htAzureConsumptionSubscriptions.($subscriptionId.Name) = @{ + ConsumptionData = $subscriptionId.group + TotalCost = $subTotalCost + Currency = $currency.Name + } + + $resourceTypes = $subscriptionId.Group.ResourceType | Sort-Object -Unique + + foreach ($parentMg in $htSubscriptionsMgPath.($subscriptionId.Name).ParentNameChain) { + + if (-not $htManagementGroupsCost.($parentMg)) { + $script:htManagementGroupsCost.($parentMg) = @{ + currencies = $currency.Name + "mgTotalCost_$($currency.Name)" = $subTotalCost #[decimal]$subTotalCost + "resourcesThatGeneratedCost_$($currency.Name)" = ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + resourcesThatGeneratedCostCurrencyIndependent = ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + "subscriptionsThatGeneratedCost_$($currency.Name)" = 1 + subscriptionsThatGeneratedCostCurrencyIndependent = 1 + "resourceTypesThatGeneratedCost_$($currency.Name)" = $resourceTypes + resourceTypesThatGeneratedCostCurrencyIndependent = $resourceTypes + "consumptionDataSubscriptions_$($currency.Name)" = $subscriptionId.group + consumptionDataSubscriptions = $subscriptionId.group + } + + } + else { + $newMgTotalCost = $htManagementGroupsCost.($parentMg)."mgTotalCost_$($currency.Name)" + $subTotalCost #[decimal]$subTotalCost + $script:htManagementGroupsCost.($parentMg)."mgTotalCost_$($currency.Name)" = $newMgTotalCost #[decimal]$newMgTotalCost + + $currencies = [array]$htManagementGroupsCost.($parentMg).currencies + if ($currencies -notcontains $currency.Name) { + $currencies += $currency.Name + $script:htManagementGroupsCost.($parentMg).currencies = $currencies + } + + #currency based + $resourcesThatGeneratedCost = $htManagementGroupsCost.($parentMg)."resourcesThatGeneratedCost_$($currency.Name)" + ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + $script:htManagementGroupsCost.($parentMg)."resourcesThatGeneratedCost_$($currency.Name)" = $resourcesThatGeneratedCost + + $subscriptionsThatGeneratedCost = $htManagementGroupsCost.($parentMg)."subscriptionsThatGeneratedCost_$($currency.Name)" + 1 + $script:htManagementGroupsCost.($parentMg)."subscriptionsThatGeneratedCost_$($currency.Name)" = $subscriptionsThatGeneratedCost + + $consumptionDataSubscriptions = $htManagementGroupsCost.($parentMg)."consumptionDataSubscriptions_$($currency.Name)" += $subscriptionId.group + $script:htManagementGroupsCost.($parentMg)."consumptionDataSubscriptions_$($currency.Name)" = $consumptionDataSubscriptions + + $resourceTypesThatGeneratedCost = $htManagementGroupsCost.($parentMg)."resourceTypesThatGeneratedCost_$($currency.Name)" + foreach ($resourceType in $resourceTypes) { + if ($resourceTypesThatGeneratedCost -notcontains $resourceType) { + $resourceTypesThatGeneratedCost += $resourceType + } + } + $script:htManagementGroupsCost.($parentMg)."resourceTypesThatGeneratedCost_$($currency.Name)" = $resourceTypesThatGeneratedCost + + #currencyIndependent + $resourcesThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).resourcesThatGeneratedCostCurrencyIndependent + ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + $script:htManagementGroupsCost.($parentMg).resourcesThatGeneratedCostCurrencyIndependent = $resourcesThatGeneratedCostCurrencyIndependent + + $subscriptionsThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).subscriptionsThatGeneratedCostCurrencyIndependent + 1 + $script:htManagementGroupsCost.($parentMg).subscriptionsThatGeneratedCostCurrencyIndependent = $subscriptionsThatGeneratedCostCurrencyIndependent + + $consumptionDataSubscriptionsCurrencyIndependent = $htManagementGroupsCost.($parentMg).consumptionDataSubscriptions += $subscriptionId.group + $script:htManagementGroupsCost.($parentMg).consumptionDataSubscriptions = $consumptionDataSubscriptionsCurrencyIndependent + + $resourceTypesThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).resourceTypesThatGeneratedCostCurrencyIndependent + foreach ($resourceType in $resourceTypes) { + if ($resourceTypesThatGeneratedCostCurrencyIndependent -notcontains $resourceType) { + $resourceTypesThatGeneratedCostCurrencyIndependent += $resourceType + } + } + $script:htManagementGroupsCost.($parentMg).resourceTypesThatGeneratedCostCurrencyIndependent = $resourceTypesThatGeneratedCostCurrencyIndependent + } + } + } + + $totalCost = 0 + $script:tenantSummaryConsumptionDataGrouped = $currency.group | Group-Object -Property ResourceType, ChargeType, MeterCategory + $subsCount = ($tenantSummaryConsumptionDataGrouped.group.subscriptionId | Sort-Object -Unique | Measure-Object).Count + $consumedServiceCount = ($tenantSummaryConsumptionDataGrouped.group.ResourceType | Sort-Object -Unique | Measure-Object).Count + $resourceCount = ($tenantSummaryConsumptionDataGrouped.group.ResourceId | Sort-Object -Unique | Measure-Object).Count + foreach ($consumptionline in $tenantSummaryConsumptionDataGrouped) { + + $costConsumptionLine = ($consumptionline.group.PreTaxCost | Measure-Object -Sum).Sum + + if ([math]::Round($costConsumptionLine, 2) -eq 0) { + $cost = $costConsumptionLine.ToString('0.0000') + } + else { + $cost = [math]::Round($costConsumptionLine, 2).ToString('0.00') + } + + $null = $script:arrayConsumptionData.Add([PSCustomObject]@{ + ResourceType = ($consumptionline.name).split(', ')[0] + ConsumedServiceChargeType = ($consumptionline.name).split(', ')[1] + ConsumedServiceCategory = ($consumptionline.name).split(', ')[2] + ConsumedServiceInstanceCount = $consumptionline.Count + ConsumedServiceCost = $cost #[decimal]$cost + ConsumedServiceSubscriptions = ($consumptionline.group.SubscriptionId | Sort-Object -Unique).Count + ConsumedServiceCurrency = $currency.Name + }) + + $totalCost = $totalCost + $costConsumptionLine + + } + if ([math]::Round($totalCost, 2) -eq 0) { + $totalCost = $totalCost + } + else { + $totalCost = [math]::Round($totalCost, 2).ToString('0.00') + } + $script:arrayTotalCostSummary += "$($totalCost) $($currency.Name) generated by $($resourceCount) Resources ($($consumedServiceCount) ResourceTypes) in $($subsCount) Subscriptions" + } + } + else { + Write-Host ' No relevant consumption data entries (0)' + } + } + + #region BuildConsumptionCSV + if (-not $NoCsvExport) { + if (-not $NoAzureConsumptionReportExportToCSV) { + Write-Host " Exporting Consumption CSV $($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" + $startBuildConsumptionCSV = Get-Date + if ($CsvExportUseQuotesAsNeeded) { + $allConsumptionData | Sort-Object -Property ResourceId | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" -Delimiter "$csvDelimiter" -NoTypeInformation -UseQuotes AsNeeded + } + else { + $allConsumptionData | Sort-Object -Property ResourceId | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + $endBuildConsumptionCSV = Get-Date + Write-Host " Exporting Consumption CSV total duration: $((New-TimeSpan -Start $startBuildConsumptionCSV -End $endBuildConsumptionCSV).TotalMinutes) minutes ($((New-TimeSpan -Start $startBuildConsumptionCSV -End $endBuildConsumptionCSV).TotalSeconds) seconds)" + } + } + #endregion BuildConsumptionCSV + } + $endConsumptionData = Get-Date + Write-Host "Getting Consumption data duration: $((New-TimeSpan -Start $startConsumptionData -End $endConsumptionData).TotalSeconds) seconds" +} function getDefaultManagementGroup { $currentTask = 'Get Default Management Group' Write-Host $currentTask @@ -5279,6 +5697,7 @@ function processALZPolicyVersionChecker { $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} @@ -5426,6 +5845,7 @@ function processALZPolicyVersionChecker { $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} @@ -5540,6 +5960,7 @@ function processALZPolicyVersionChecker { $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 67c10156..9ed3a346 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.8', + $ProductVersion = '6.4.9', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -702,6 +702,7 @@ if ($ManagementGroupId -match ' ') { . ".\$($ScriptPath)\functions\getOrphanedResources.ps1" . ".\$($ScriptPath)\functions\getMDfCSecureScoreMG.ps1" . ".\$($ScriptPath)\functions\getConsumption.ps1" +. ".\$($ScriptPath)\functions\getConsumptionv2.ps1" . ".\$($ScriptPath)\functions\cacheBuiltIn.ps1" . ".\$($ScriptPath)\functions\prepareData.ps1" . ".\$($ScriptPath)\functions\getGroupmembers.ps1" diff --git a/pwsh/dev/functions/getConsumptionv2.ps1 b/pwsh/dev/functions/getConsumptionv2.ps1 new file mode 100644 index 00000000..f5f21dda --- /dev/null +++ b/pwsh/dev/functions/getConsumptionv2.ps1 @@ -0,0 +1,418 @@ +function getConsumptionv2 { + #todo: remove + Write-Host '#########-----------------#########' -ForegroundColor DarkMagenta + Write-Host 'Executing getConsumptionv2' -ForegroundColor DarkMagenta + + $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) + + function addToAllConsumptionData { + [CmdletBinding()]Param( + [Parameter(Mandatory)] + [object] + $consumptiondataFromAPI, + + [Parameter(Mandatory)] + [string] + $subscriptionQuotaId + ) + + foreach ($consumptionline in $consumptiondataFromAPI.properties.rows) { + $hlper = $htSubscriptionsMgPath.($consumptionline[1]) + + $null = $script:allConsumptionData.Add([PSCustomObject]@{ + "$($consumptiondataFromAPI.properties.columns.name[0])" = [decimal]$consumptionline[0] + "$($consumptiondataFromAPI.properties.columns.name[1])" = $consumptionline[1] + SubscriptionName = $hlper.DisplayName + subscriptionQuotaId = $subscriptionQuotaId + SubscriptionMgPath = $hlper.ParentNameChainDelimited + "$($consumptiondataFromAPI.properties.columns.name[2])" = $consumptionline[2] + "$($consumptiondataFromAPI.properties.columns.name[3])" = $consumptionline[3] + "$($consumptiondataFromAPI.properties.columns.name[4])" = $consumptionline[4] + "$($consumptiondataFromAPI.properties.columns.name[5])" = $consumptionline[5] + "$($consumptiondataFromAPI.properties.columns.name[6])" = $consumptionline[6] + }) + } + } + + $startConsumptionData = Get-Date + + if ($subsToProcessInCustomDataCollectionCount -gt 0) { + + #$subscriptionIdsOptimizedForBody = '"{0}"' -f ($subsToProcessInCustomDataCollection.subscriptionId -join '","') + $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" + #https://learn.microsoft.com/rest/api/cost-management/query/usage + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=100" + $method = 'POST' + + $subsToProcessInCustomDataCollectionGroupedByQuotaId = $subsToProcessInCustomDataCollection | Group-Object -Property subscriptionQuotaId + $cnter = 0 + foreach ($quotaIdGroup in $subsToProcessInCustomDataCollectionGroupedByQuotaId) { + + $counterBatch = [PSCustomObject] @{ Value = 0 } + $batchSize = 100 + $subscriptionsBatch = ($quotaIdGroup.Group) | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } + $batchCnt = 0 + Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId $($quotaIdGroup.Name) in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" + + foreach ($batch in $subscriptionsBatch) { + $cnter++ + $batchCnt++ + $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') + $currenttask = " Getting Consumption data quotaId:'$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" -ForegroundColor Cyan + + $body = @" + { + "type": "ActualCost", + "dataset": { + "granularity": "none", + "filter": { + "dimensions": { + "name": "SubscriptionId", + "operator": "In", + "values": [ + $($subscriptionIdsOptimizedForBody) + ] + } + }, + "aggregation": { + "totalCost": { + "name": "PreTaxCost", + "function": "Sum" + } + }, + "grouping": [ + { + "type": "Dimension", + "name": "SubscriptionId" + }, + { + "type": "Dimension", + "name": "ResourceId" + }, + { + "type": "Dimension", + "name": "ResourceType" + }, + { + "type": "Dimension", + "name": "MeterCategory" + }, + { + "type": "Dimension", + "name": "ChargeType" + } + ] + }, + "timeframe": "Custom", + "timeperiod": { + "from": "$($azureConsumptionStartDate)", + "to": "$($azureConsumptionEndDate)" + } + } +"@ + + $mgConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + + <#test + #$mgConsumptionData = "OfferNotSupported" + if ($batchCnt -eq 1){ + $mgConsumptionData = "OfferNotSupported" + } + #> + #enforce switch to 'foreach Subscription' Subscription scope mode + # if ($cnter -eq 2) { + # $mgConsumptionData = 'Unauthorized' + # } + if ($mgConsumptionData -eq 'Unauthorized' -or $mgConsumptionData -eq 'OfferNotSupported' -or $mgConsumptionData -eq 'NoValidSubscriptions') { + if (-not $script:htConsumptionExceptionLog.Mg.($ManagementGroupId)) { + $script:htConsumptionExceptionLog.Mg.($ManagementGroupId) = @{} + } + $script:htConsumptionExceptionLog.Mg.($ManagementGroupId).($batchCnt) = @{ + Exception = $mgConsumptionData + Subscriptions = ($batch.Group).subscriptionId + } + + Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data #batch$($batchCnt) using Management Group scope failed." + $body = @" + { + "type": "ActualCost", + "dataset": { + "granularity": "none", + "aggregation": { + "totalCost": { + "name": "PreTaxCost", + "function": "Sum" + } + }, + "grouping": [ + { + "type": "Dimension", + "name": "SubscriptionId" + }, + { + "type": "Dimension", + "name": "ResourceId" + }, + { + "type": "Dimension", + "name": "ResourceType" + }, + { + "type": "Dimension", + "name": "MeterCategory" + }, + { + "type": "Dimension", + "name": "ChargeType" + } + ] + }, + "timeframe": "Custom", + "timeperiod": { + "from": "$($azureConsumptionStartDate)", + "to": "$($azureConsumptionEndDate)" + } + } +"@ + $funcAddToAllConsumptionData = $function:addToAllConsumptionData.ToString() + $batch.Group | ForEach-Object -Parallel { + $subIdToProcess = $_.subscriptionId + $subNameToProcess = $_.subscriptionName + $subQuotaId = $_.subscriptionQuotaId + #region UsingVARs + $body = $using:body + $azureConsumptionStartDate = $using:azureConsumptionStartDate + $azureConsumptionEndDate = $using:azureConsumptionEndDate + #fromOtherFunctions + $azAPICallConf = $using:azAPICallConf + $scriptPath = $using:ScriptPath + #Array&HTs + $allConsumptionData = $using:allConsumptionData + $htSubscriptionsMgPath = $using:htSubscriptionsMgPath + $htAllSubscriptionsFromAPI = $using:htAllSubscriptionsFromAPI + $htConsumptionExceptionLog = $using:htConsumptionExceptionLog + #other + $function:addToAllConsumptionData = $using:funcAddToAllConsumptionData + $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion + #endregion UsingVARs + + $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + #test + Write-Host $currentTask + #https://learn.microsoft.com/rest/api/cost-management/query/usage + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" + $method = 'POST' + $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { + Write-Host " Failed ($subConsumptionData) - Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails + $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) + $script:htConsumptionExceptionLog.Sub.($subIdToProcess) = @{ + Exception = $subConsumptionData + SubscriptionId = $subIdToProcess + SubscriptionName = $hlper.displayName + QuotaId = $hlper.subscriptionPolicies.quotaId + mgPath = $hlper2.ParentNameChainDelimited + mgParent = $hlper2.Parent + } + + Continue + } + else { + Write-Host " $($subConsumptionData.properties.rows.Count) Consumption data entries (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + if ($subConsumptionData.Count -gt 0) { + addToAllConsumptionData -consumptiondataFromAPI $subConsumptionData -subscriptionQuotaId $subQuotaId + } + } + } -ThrottleLimit $ThrottleLimit + } + else { + Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" + if ($mgConsumptionData.Count -gt 0) { + addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name + } + } + } + } + + } + else { + $detailShowStopperResult = 'NoSubscriptionsPresent' + Write-Host ' No Subscriptions present, skipping Consumption data processing' + } + + + if ($detailShowStopperResult -eq 'AccountCostDisabled' -or $detailShowStopperResult -eq 'NoValidSubscriptions' -or $detailShowStopperResult -eq 'NoSubscriptionsPresent') { + if ($detailShowStopperResult -eq 'AccountCostDisabled') { + Write-Host ' Seems Access to cost data has been disabled for this Account - skipping CostManagement' + } + if ($detailShowStopperResult -eq 'NoValidSubscriptions') { + Write-Host ' Seems there are no valid Subscriptions present - skipping CostManagement' + } + if ($detailShowStopperResult -eq 'NoSubscriptionsPresent') { + Write-Host ' Seems there are no Subscriptions present - skipping CostManagement' + } + Write-Host " Action: Setting switch parameter 'DoAzureConsumption' to false" + $azAPICallConf['htParameters'].DoAzureConsumption = $false + } + else { + Write-Host ' Checking returned Consumption data' + $script:allConsumptionDataCount = $allConsumptionData.Count + + if ($allConsumptionDataCount -gt 0) { + + $script:allConsumptionData = $allConsumptionData.where( { $_.PreTaxCost -ne 0 } ) + $script:allConsumptionDataCount = $allConsumptionData.Count + + if ($allConsumptionDataCount -gt 0) { + Write-Host " $($allConsumptionDataCount) relevant Consumption data entries" + + $script:consumptionData = $allConsumptionData + $script:consumptionDataGroupedByCurrency = $consumptionData | Group-Object -Property Currency + + foreach ($currency in $consumptionDataGroupedByCurrency) { + + #subscriptions + $groupAllConsumptionDataPerCurrencyBySubscriptionId = $currency.group | Group-Object -Property SubscriptionId + foreach ($subscriptionId in $groupAllConsumptionDataPerCurrencyBySubscriptionId) { + + $subTotalCost = ($subscriptionId.Group.PreTaxCost | Measure-Object -Sum).Sum + $script:htAzureConsumptionSubscriptions.($subscriptionId.Name) = @{ + ConsumptionData = $subscriptionId.group + TotalCost = $subTotalCost + Currency = $currency.Name + } + + $resourceTypes = $subscriptionId.Group.ResourceType | Sort-Object -Unique + + foreach ($parentMg in $htSubscriptionsMgPath.($subscriptionId.Name).ParentNameChain) { + + if (-not $htManagementGroupsCost.($parentMg)) { + $script:htManagementGroupsCost.($parentMg) = @{ + currencies = $currency.Name + "mgTotalCost_$($currency.Name)" = $subTotalCost #[decimal]$subTotalCost + "resourcesThatGeneratedCost_$($currency.Name)" = ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + resourcesThatGeneratedCostCurrencyIndependent = ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + "subscriptionsThatGeneratedCost_$($currency.Name)" = 1 + subscriptionsThatGeneratedCostCurrencyIndependent = 1 + "resourceTypesThatGeneratedCost_$($currency.Name)" = $resourceTypes + resourceTypesThatGeneratedCostCurrencyIndependent = $resourceTypes + "consumptionDataSubscriptions_$($currency.Name)" = $subscriptionId.group + consumptionDataSubscriptions = $subscriptionId.group + } + + } + else { + $newMgTotalCost = $htManagementGroupsCost.($parentMg)."mgTotalCost_$($currency.Name)" + $subTotalCost #[decimal]$subTotalCost + $script:htManagementGroupsCost.($parentMg)."mgTotalCost_$($currency.Name)" = $newMgTotalCost #[decimal]$newMgTotalCost + + $currencies = [array]$htManagementGroupsCost.($parentMg).currencies + if ($currencies -notcontains $currency.Name) { + $currencies += $currency.Name + $script:htManagementGroupsCost.($parentMg).currencies = $currencies + } + + #currency based + $resourcesThatGeneratedCost = $htManagementGroupsCost.($parentMg)."resourcesThatGeneratedCost_$($currency.Name)" + ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + $script:htManagementGroupsCost.($parentMg)."resourcesThatGeneratedCost_$($currency.Name)" = $resourcesThatGeneratedCost + + $subscriptionsThatGeneratedCost = $htManagementGroupsCost.($parentMg)."subscriptionsThatGeneratedCost_$($currency.Name)" + 1 + $script:htManagementGroupsCost.($parentMg)."subscriptionsThatGeneratedCost_$($currency.Name)" = $subscriptionsThatGeneratedCost + + $consumptionDataSubscriptions = $htManagementGroupsCost.($parentMg)."consumptionDataSubscriptions_$($currency.Name)" += $subscriptionId.group + $script:htManagementGroupsCost.($parentMg)."consumptionDataSubscriptions_$($currency.Name)" = $consumptionDataSubscriptions + + $resourceTypesThatGeneratedCost = $htManagementGroupsCost.($parentMg)."resourceTypesThatGeneratedCost_$($currency.Name)" + foreach ($resourceType in $resourceTypes) { + if ($resourceTypesThatGeneratedCost -notcontains $resourceType) { + $resourceTypesThatGeneratedCost += $resourceType + } + } + $script:htManagementGroupsCost.($parentMg)."resourceTypesThatGeneratedCost_$($currency.Name)" = $resourceTypesThatGeneratedCost + + #currencyIndependent + $resourcesThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).resourcesThatGeneratedCostCurrencyIndependent + ($subscriptionId.Group.ResourceId | Sort-Object -Unique | Measure-Object).Count + $script:htManagementGroupsCost.($parentMg).resourcesThatGeneratedCostCurrencyIndependent = $resourcesThatGeneratedCostCurrencyIndependent + + $subscriptionsThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).subscriptionsThatGeneratedCostCurrencyIndependent + 1 + $script:htManagementGroupsCost.($parentMg).subscriptionsThatGeneratedCostCurrencyIndependent = $subscriptionsThatGeneratedCostCurrencyIndependent + + $consumptionDataSubscriptionsCurrencyIndependent = $htManagementGroupsCost.($parentMg).consumptionDataSubscriptions += $subscriptionId.group + $script:htManagementGroupsCost.($parentMg).consumptionDataSubscriptions = $consumptionDataSubscriptionsCurrencyIndependent + + $resourceTypesThatGeneratedCostCurrencyIndependent = $htManagementGroupsCost.($parentMg).resourceTypesThatGeneratedCostCurrencyIndependent + foreach ($resourceType in $resourceTypes) { + if ($resourceTypesThatGeneratedCostCurrencyIndependent -notcontains $resourceType) { + $resourceTypesThatGeneratedCostCurrencyIndependent += $resourceType + } + } + $script:htManagementGroupsCost.($parentMg).resourceTypesThatGeneratedCostCurrencyIndependent = $resourceTypesThatGeneratedCostCurrencyIndependent + } + } + } + + $totalCost = 0 + $script:tenantSummaryConsumptionDataGrouped = $currency.group | Group-Object -Property ResourceType, ChargeType, MeterCategory + $subsCount = ($tenantSummaryConsumptionDataGrouped.group.subscriptionId | Sort-Object -Unique | Measure-Object).Count + $consumedServiceCount = ($tenantSummaryConsumptionDataGrouped.group.ResourceType | Sort-Object -Unique | Measure-Object).Count + $resourceCount = ($tenantSummaryConsumptionDataGrouped.group.ResourceId | Sort-Object -Unique | Measure-Object).Count + foreach ($consumptionline in $tenantSummaryConsumptionDataGrouped) { + + $costConsumptionLine = ($consumptionline.group.PreTaxCost | Measure-Object -Sum).Sum + + if ([math]::Round($costConsumptionLine, 2) -eq 0) { + $cost = $costConsumptionLine.ToString('0.0000') + } + else { + $cost = [math]::Round($costConsumptionLine, 2).ToString('0.00') + } + + $null = $script:arrayConsumptionData.Add([PSCustomObject]@{ + ResourceType = ($consumptionline.name).split(', ')[0] + ConsumedServiceChargeType = ($consumptionline.name).split(', ')[1] + ConsumedServiceCategory = ($consumptionline.name).split(', ')[2] + ConsumedServiceInstanceCount = $consumptionline.Count + ConsumedServiceCost = $cost #[decimal]$cost + ConsumedServiceSubscriptions = ($consumptionline.group.SubscriptionId | Sort-Object -Unique).Count + ConsumedServiceCurrency = $currency.Name + }) + + $totalCost = $totalCost + $costConsumptionLine + + } + if ([math]::Round($totalCost, 2) -eq 0) { + $totalCost = $totalCost + } + else { + $totalCost = [math]::Round($totalCost, 2).ToString('0.00') + } + $script:arrayTotalCostSummary += "$($totalCost) $($currency.Name) generated by $($resourceCount) Resources ($($consumedServiceCount) ResourceTypes) in $($subsCount) Subscriptions" + } + } + else { + Write-Host ' No relevant consumption data entries (0)' + } + } + + #region BuildConsumptionCSV + if (-not $NoCsvExport) { + if (-not $NoAzureConsumptionReportExportToCSV) { + Write-Host " Exporting Consumption CSV $($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" + $startBuildConsumptionCSV = Get-Date + if ($CsvExportUseQuotesAsNeeded) { + $allConsumptionData | Sort-Object -Property ResourceId | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" -Delimiter "$csvDelimiter" -NoTypeInformation -UseQuotes AsNeeded + } + else { + $allConsumptionData | Sort-Object -Property ResourceId | Export-Csv -Path "$($outputPath)$($DirectorySeparatorChar)$($fileName)_Consumption.csv" -Delimiter "$csvDelimiter" -NoTypeInformation + } + $endBuildConsumptionCSV = Get-Date + Write-Host " Exporting Consumption CSV total duration: $((New-TimeSpan -Start $startBuildConsumptionCSV -End $endBuildConsumptionCSV).TotalMinutes) minutes ($((New-TimeSpan -Start $startBuildConsumptionCSV -End $endBuildConsumptionCSV).TotalSeconds) seconds)" + } + } + #endregion BuildConsumptionCSV + } + $endConsumptionData = Get-Date + Write-Host "Getting Consumption data duration: $((New-TimeSpan -Start $startConsumptionData -End $endConsumptionData).TotalSeconds) seconds" +} \ No newline at end of file diff --git a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 index 12424869..121332d1 100644 --- a/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 +++ b/pwsh/dev/functions/processALZPolicyVersionChecker.ps1 @@ -263,6 +263,7 @@ $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} @@ -410,6 +411,7 @@ $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} @@ -524,6 +526,7 @@ $hashPolicyDefinitions = [System.Security.Cryptography.HashAlgorithm]::Create('sha256').ComputeHash([System.Text.Encoding]::UTF8.GetBytes($policyJsonPolicyDefinitions)) $stringHashPolicyDefinitions = [System.BitConverter]::ToString($hashPolicyDefinitions) $stringHash = "$($stringHashParameters)_$($stringHashPolicyDefinitions)" + #($policyJsonRebuild.properties | ConvertTo-Json -Depth 99) > "c:\temp\alz3\$($policyJsonRebuild.name)_$($policyJsonRebuild.properties.metadata.version).json" if (-not $allESLZPolicySets.($policyJsonRebuild.name)) { $allESLZPolicySets.($policyJsonRebuild.name) = @{} diff --git a/version.json b/version.json index b642bbbb..c77cb0f1 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.8" + "ProductVersion": "6.4.9" } \ No newline at end of file From 5fa4361e09e89606fb0bb401d313747a8e4f8b03 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Tue, 18 Jun 2024 19:46:52 +0200 Subject: [PATCH 16/20] 6.4.10 --- README.md | 3 ++- history.md | 3 ++- pwsh/AzGovVizParallel.ps1 | 15 ++++++++++----- pwsh/dev/devAzGovVizParallel.ps1 | 15 ++++++++++----- version.json | 2 +- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 68dd29cc..b5a8f09f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt ## Release history -**Changes** (2024-June-18 / 6.4.9 Minor) +**Changes** (2024-June-18 / 6.4.10 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -92,6 +92,7 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) - update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- html; update jquery; source tablefilter js [Full release history](history.md) diff --git a/history.md b/history.md index 1481e6f9..214c910e 100644 --- a/history.md +++ b/history.md @@ -4,7 +4,7 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-June-18 / 6.4.9 Minor) +**Changes** (2024-June-18 / 6.4.10 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -14,6 +14,7 @@ - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) - update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- html; update jquery; source tablefilter js **Changes** (2024-May-05 / 6.4.5 Minor) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 30e60a91..00eb9b4a 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.9', + $ProductVersion = '6.4.10', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -36063,13 +36063,18 @@ $html = @" document.getElementsByTagName( "head" )[0].appendChild( link ); - - + + + + - + + + - + + diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 9ed3a346..6a01719d 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.9', + $ProductVersion = '6.4.10', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -1794,13 +1794,18 @@ $html = @" document.getElementsByTagName( "head" )[0].appendChild( link ); - - + + + + - + + + - + + diff --git a/version.json b/version.json index c77cb0f1..963438f6 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.9" + "ProductVersion": "6.4.10" } \ No newline at end of file From 97c78b6d0a3fe83a6397ac9cf9de2844b515d5b4 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 15 Jul 2024 12:16:40 +0200 Subject: [PATCH 17/20] 6.4.11 --- .devcontainer/devcontainer.json | 22 ++++---- .github/workflows/psScriptAnalyzer.yml | 2 +- README.md | 14 ++--- history.md | 5 +- pwsh/AzGovVizParallel.ps1 | 68 ++++++++++++++++--------- pwsh/dev/devAzGovVizParallel.ps1 | 7 ++- pwsh/dev/functions/getConsumptionv2.ps1 | 60 ++++++++++++++-------- pwsh/dev/functions/verifyModules3rd.ps1 | 1 - version.json | 2 +- 9 files changed, 116 insertions(+), 65 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 72c90fc1..cfccf5d8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,15 +1,19 @@ { "name": "AzureGovernanceVisualizer", "dockerFile": "Dockerfile", - "settings": { - "terminal.integrated.defaultProfile.linux": "pwsh" + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.defaultProfile.linux": "pwsh" + }, + "extensions": [ + "ms-vscode.powershell", + "analytic-signal.preview-html", + "bierner.markdown-mermaid", + "streetsidesoftware.code-spell-checker", + "yzhang.markdown-all-in-one" + ] + } }, - "extensions": [ - "ms-vscode.powershell", - "analytic-signal.preview-html", - "bierner.markdown-mermaid", - "streetsidesoftware.code-spell-checker", - "yzhang.markdown-all-in-one" - ], "forwardPorts": [] } \ No newline at end of file diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index 4c7d852d..bd7ec582 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -41,7 +41,7 @@ jobs: recurse: true # Include your own basic security rules. Removing this option will run all the rules # includeRule: '"PSAvoidGlobalAliases", "PSAvoidUsingConvertToSecureStringWithPlainText"' - excludeRule: '"PSAvoidUsingWriteHost", "PSUseDeclaredVarsMoreThanAssignments", "PSReviewUnusedParameter"' + excludeRule: '"PSAvoidUsingWriteHost", "PSUseDeclaredVarsMoreThanAssignments", "PSReviewUnusedParameter", "PSUseOutputTypeCorrectly"' output: results.sarif # Upload the SARIF file generated in the previous step diff --git a/README.md b/README.md index b5a8f09f..9d0f4f5b 100644 --- a/README.md +++ b/README.md @@ -70,19 +70,19 @@ Azure Governance Visualizer is intended to help you to get a holistic overview o - Listed as [tool](https://learn.microsoft.com/azure/cloud-adoption-framework/resources/tools-templates#govern) for the Govern discipline in the Microsoft Cloud Adoption Framework. - Included in the Cloud Adoption Framework's [Strategy-Plan-Ready-Governance](https://azuredevopsdemogenerator.azurewebsites.net/?name=strategyplan) Azure DevOps Demo Generator template. -### Azure Governance Visualizer accelerator - -The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Governance-Visualizer-Accelerator) provides an easy and fast deployment process that automates the creation and publishing of AzGovViz to an Azure Web Application and provides automation to configuring the pre-requisites for AzGovViz. - ## :rocket: Azure Governance Visualizer deployment guide The instructions to deploy the Azure Governance Visualizer is found in the **[Azure Governance Visualizer (AzGovViz) deployment guide](setup.md)**. Follow those instructions to run AzGovViz from your terminal (console), GitHub Codepaces, Azure DevOps, or GitHub. As an alternative, you can use the [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Governance-Visualizer-Accelerator) to deploy the Azure Governance Visualizer per code. +### Azure Governance Visualizer accelerator + +The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Governance-Visualizer-Accelerator) provides an easy and fast deployment process that automates the creation and publishing of AzGovViz to an Azure Web Application and provides automation to configuring the pre-requisites for AzGovViz. + ## Release history -**Changes** (2024-June-18 / 6.4.10 Minor) +**Changes** (2024-July-15 / 6.4.11 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -91,8 +91,9 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt - update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) -- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` - html; update jquery; source tablefilter js +- update `.devcontainer/devcontainer.json` [Full release history](history.md) @@ -109,6 +110,7 @@ More [demo output](https://github.com/JulianHayward/AzGovViz) - Microsoft Tech Talks - Bevan Sinclair (Cloud Solution Architect Microsoft) [Automated Governance Reporting in Azure (MTT0AEDT)](https://mtt.eventbuilder.com/event/66431) (register to view) - Microsoft Dev Radio (YouTube) [Get visibility into your environment with Azure Governance Visualizer](https://www.youtube.com/watch?v=hZXvF5oypLE) - Jack Tracey (Cloud Solution Architect Microsoft) [Azure Governance Visualizer With Azure DevOps](https://jacktracey.co.uk/azgovviz-with-azure-devops/) +- SCHUTTEN.CLOUD [Automate Pertinent Governance Insight with Azure Governance Visualizer](https://schutten.cloud/post/azure-governance-visualizer/) ### Presentations diff --git a/history.md b/history.md index 214c910e..b1ebbc7a 100644 --- a/history.md +++ b/history.md @@ -4,7 +4,7 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-June-18 / 6.4.10 Minor) +**Changes** (2024-July-15 / 6.4.11 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -13,8 +13,9 @@ - update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) -- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` - html; update jquery; source tablefilter js +- update `.devcontainer/devcontainer.json` **Changes** (2024-May-05 / 6.4.5 Minor) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 00eb9b4a..0d0a3d49 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.10', + $ProductVersion = '6.4.11', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -646,7 +646,10 @@ Param AzureUSGovernment = '2023-01-01' AzureChinaCloud = '2023-01-01' } - } + }, + + [array] + $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #PayAsYouGo_2014-09-01 ) $Error.clear() @@ -3374,13 +3377,10 @@ function getConsumptionv2 { $startConsumptionData = Get-Date if ($subsToProcessInCustomDataCollectionCount -gt 0) { - - #$subscriptionIdsOptimizedForBody = '"{0}"' -f ($subsToProcessInCustomDataCollection.subscriptionId -join '","') - $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + $currenttask = "Getting Consumption data scope MG (ManagementGroupId '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=100" - $method = 'POST' + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $subsToProcessInCustomDataCollectionGroupedByQuotaId = $subsToProcessInCustomDataCollection | Group-Object -Property subscriptionQuotaId $cnter = 0 @@ -3390,16 +3390,22 @@ function getConsumptionv2 { $batchSize = 100 $subscriptionsBatch = ($quotaIdGroup.Group) | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } $batchCnt = 0 - Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId $($quotaIdGroup.Name) in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" + Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId '$($quotaIdGroup.Name)' in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" foreach ($batch in $subscriptionsBatch) { $cnter++ $batchCnt++ - $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') - $currenttask = " Getting Consumption data quotaId:'$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" - Write-Host "$currentTask" -ForegroundColor Cyan + if ($quotaIdGroup.Name -in $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery) { + Write-Host " Enforcing 'foreach Subscription' Subscription scope mode, due to QuotaId '$($quotaIdGroup.Name)' for $($batch.Group.Count) Subscriptions" + $mgConsumptionData = 'NoValidSubscriptions' + } + else { + $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') + $currenttask = " Getting Consumption data QuotaId '$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" -ForegroundColor Cyan - $body = @" + + $bodyMGScope = @" { "type": "ActualCost", "dataset": { @@ -3450,7 +3456,17 @@ function getConsumptionv2 { } "@ - $mgConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + $mgConsumptionDataParametersSplat = @{ + AzAPICallConfiguration = $azAPICallConf + uri = $uri + method = 'POST' + body = $bodyMGScope + currentTask = $currentTask + listenOn = 'ContentProperties' + } + $mgConsumptionData = AzAPICall @mgConsumptionDataParametersSplat + + } <#test #$mgConsumptionData = "OfferNotSupported" @@ -3471,8 +3487,8 @@ function getConsumptionv2 { Subscriptions = ($batch.Group).subscriptionId } - Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data #batch$($batchCnt) using Management Group scope failed." - $body = @" + Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count)" + $bodySubScope = @" { "type": "ActualCost", "dataset": { @@ -3519,7 +3535,7 @@ function getConsumptionv2 { $subNameToProcess = $_.subscriptionName $subQuotaId = $_.subscriptionQuotaId #region UsingVARs - $body = $using:body + $bodySubScope = $using:bodySubScope $azureConsumptionStartDate = $using:azureConsumptionStartDate $azureConsumptionEndDate = $using:azureConsumptionEndDate #fromOtherFunctions @@ -3535,15 +3551,23 @@ function getConsumptionv2 { $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs - $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + $currentTask = " Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" - $method = 'POST' - $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + $subConsumptionDataParametersSplat = @{ + AzAPICallConfiguration = $azAPICallConf + uri = $uri + method = 'POST' + body = $bodySubScope + currentTask = $currentTask + listenOn = 'ContentProperties' + } + $subConsumptionData = AzAPICall @subConsumptionDataParametersSplat + if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { - Write-Host " Failed ($subConsumptionData) - Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + Write-Host " Failed ($subConsumptionData) - Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) $script:htConsumptionExceptionLog.Sub.($subIdToProcess) = @{ @@ -3566,14 +3590,13 @@ function getConsumptionv2 { } -ThrottleLimit $ThrottleLimit } else { - Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" + Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" if ($mgConsumptionData.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name } } } } - } else { $detailShowStopperResult = 'NoSubscriptionsPresent' @@ -30423,7 +30446,6 @@ function verifyModules3rd { if (-not $installAzAPICallModuleSuccess) { throw " Installing '$($module.ModuleName)' module ($($moduleVersion)) failed" } - } else { do { diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 6a01719d..01d0267b 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.10', + $ProductVersion = '6.4.11', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -646,7 +646,10 @@ Param AzureUSGovernment = '2023-01-01' AzureChinaCloud = '2023-01-01' } - } + }, + + [array] + $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #PayAsYouGo_2014-09-01 ) $Error.clear() diff --git a/pwsh/dev/functions/getConsumptionv2.ps1 b/pwsh/dev/functions/getConsumptionv2.ps1 index f5f21dda..7fa0df7c 100644 --- a/pwsh/dev/functions/getConsumptionv2.ps1 +++ b/pwsh/dev/functions/getConsumptionv2.ps1 @@ -37,13 +37,10 @@ $startConsumptionData = Get-Date if ($subsToProcessInCustomDataCollectionCount -gt 0) { - - #$subscriptionIdsOptimizedForBody = '"{0}"' -f ($subsToProcessInCustomDataCollection.subscriptionId -join '","') - $currenttask = "Getting Consumption data (scope MG '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + $currenttask = "Getting Consumption data scope MG (ManagementGroupId '$($ManagementGroupId)') for $($subsToProcessInCustomDataCollectionCount) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" Write-Host "$currentTask" #https://learn.microsoft.com/rest/api/cost-management/query/usage - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=100" - $method = 'POST' + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.Management/managementGroups/$($ManagementGroupId)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" $subsToProcessInCustomDataCollectionGroupedByQuotaId = $subsToProcessInCustomDataCollection | Group-Object -Property subscriptionQuotaId $cnter = 0 @@ -53,16 +50,22 @@ $batchSize = 100 $subscriptionsBatch = ($quotaIdGroup.Group) | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) } $batchCnt = 0 - Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId $($quotaIdGroup.Name) in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" + Write-Host " Processing $($quotaIdGroup.Count) Subscriptions with QuotaId '$($quotaIdGroup.Name)' in $(($subscriptionsBatch | Measure-Object).Count) batch(es) of max $batchSize Subscriptions" foreach ($batch in $subscriptionsBatch) { $cnter++ $batchCnt++ - $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') - $currenttask = " Getting Consumption data quotaId:'$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" - Write-Host "$currentTask" -ForegroundColor Cyan + if ($quotaIdGroup.Name -in $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery) { + Write-Host " Enforcing 'foreach Subscription' Subscription scope mode, due to QuotaId '$($quotaIdGroup.Name)' for $($batch.Group.Count) Subscriptions" + $mgConsumptionData = 'NoValidSubscriptions' + } + else { + $subscriptionIdsOptimizedForBody = '"{0}"' -f (($batch.Group).subscriptionId -join '","') + $currenttask = " Getting Consumption data QuotaId '$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) (scope MG '$($ManagementGroupId)') for $(($batch.Group).Count) Subscriptions for period $AzureConsumptionPeriod days ($azureConsumptionStartDate - $azureConsumptionEndDate)" + Write-Host "$currentTask" -ForegroundColor Cyan - $body = @" + + $bodyMGScope = @" { "type": "ActualCost", "dataset": { @@ -113,7 +116,17 @@ } "@ - $mgConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + $mgConsumptionDataParametersSplat = @{ + AzAPICallConfiguration = $azAPICallConf + uri = $uri + method = 'POST' + body = $bodyMGScope + currentTask = $currentTask + listenOn = 'ContentProperties' + } + $mgConsumptionData = AzAPICall @mgConsumptionDataParametersSplat + + } <#test #$mgConsumptionData = "OfferNotSupported" @@ -134,8 +147,8 @@ Subscriptions = ($batch.Group).subscriptionId } - Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data #batch$($batchCnt) using Management Group scope failed." - $body = @" + Write-Host " Switching to 'foreach Subscription' Subscription scope mode. Getting Consumption data for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count)" + $bodySubScope = @" { "type": "ActualCost", "dataset": { @@ -182,7 +195,7 @@ $subNameToProcess = $_.subscriptionName $subQuotaId = $_.subscriptionQuotaId #region UsingVARs - $body = $using:body + $bodySubScope = $using:bodySubScope $azureConsumptionStartDate = $using:azureConsumptionStartDate $azureConsumptionEndDate = $using:azureConsumptionEndDate #fromOtherFunctions @@ -198,15 +211,23 @@ $costManagementQueryAPIVersion = $using:costManagementQueryAPIVersion #endregion UsingVARs - $currentTask = " Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + $currentTask = " Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" #test Write-Host $currentTask #https://learn.microsoft.com/rest/api/cost-management/query/usage $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subIdToProcess)/providers/Microsoft.CostManagement/query?api-version=$($costManagementQueryAPIVersion)&`$top=5000" - $method = 'POST' - $subConsumptionData = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -body $body -currentTask $currentTask -listenOn 'ContentProperties' + $subConsumptionDataParametersSplat = @{ + AzAPICallConfiguration = $azAPICallConf + uri = $uri + method = 'POST' + body = $bodySubScope + currentTask = $currentTask + listenOn = 'ContentProperties' + } + $subConsumptionData = AzAPICall @subConsumptionDataParametersSplat + if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { - Write-Host " Failed ($subConsumptionData) - Getting Consumption data (scope Sub $($subNameToProcess) '$($subIdToProcess)')" + Write-Host " Failed ($subConsumptionData) - Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) $script:htConsumptionExceptionLog.Sub.($subIdToProcess) = @{ @@ -229,14 +250,13 @@ } -ThrottleLimit $ThrottleLimit } else { - Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" + Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" if ($mgConsumptionData.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name } } } } - } else { $detailShowStopperResult = 'NoSubscriptionsPresent' diff --git a/pwsh/dev/functions/verifyModules3rd.ps1 b/pwsh/dev/functions/verifyModules3rd.ps1 index 21683b5e..0b0a8115 100644 --- a/pwsh/dev/functions/verifyModules3rd.ps1 +++ b/pwsh/dev/functions/verifyModules3rd.ps1 @@ -93,7 +93,6 @@ if (-not $installAzAPICallModuleSuccess) { throw " Installing '$($module.ModuleName)' module ($($moduleVersion)) failed" } - } else { do { diff --git a/version.json b/version.json index 963438f6..1b1373f4 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.10" + "ProductVersion": "6.4.11" } \ No newline at end of file From f56ab31fe8268982a23c6b9d79ef7b504049a9dc Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 15 Jul 2024 22:11:15 +0200 Subject: [PATCH 18/20] 6.4.12; azAPICall 1.2.3 --- README.md | 3 ++- history.md | 3 ++- pwsh/AzGovVizParallel.ps1 | 20 +++++++++++++++----- pwsh/dev/devAzGovVizParallel.ps1 | 4 ++-- pwsh/dev/functions/getConsumptionv2.ps1 | 16 +++++++++++++--- version.json | 2 +- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9d0f4f5b..a708cac1 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Gov ## Release history -**Changes** (2024-July-15 / 6.4.11 Minor) +**Changes** (2024-July-15 / 6.4.12 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -94,6 +94,7 @@ The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Gov - update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` - html; update jquery; source tablefilter js - update `.devcontainer/devcontainer.json` +- use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.3 (Handle costManagement error `SubscriptionCostDisabled`) [Full release history](history.md) diff --git a/history.md b/history.md index b1ebbc7a..ae55fa80 100644 --- a/history.md +++ b/history.md @@ -4,7 +4,7 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-July-15 / 6.4.11 Minor) +**Changes** (2024-July-15 / 6.4.12 Minor) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -16,6 +16,7 @@ - update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` - html; update jquery; source tablefilter js - update `.devcontainer/devcontainer.json` +- use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.3 (Handle costManagement error `SubscriptionCostDisabled`) **Changes** (2024-May-05 / 6.4.5 Minor) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 0d0a3d49..1ad15d61 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,14 +365,14 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.11', + $ProductVersion = '6.4.12', [string] $GithubRepository = 'aka.ms/AzGovViz', # <--- AzAPICall related parameters #consult the AzAPICall GitHub repository for details aka.ms/AzAPICall [string] - $AzAPICallVersion = '1.2.1', + $AzAPICallVersion = '1.2.3', [switch] $DebugAzAPICall, @@ -3566,7 +3566,17 @@ function getConsumptionv2 { } $subConsumptionData = AzAPICall @subConsumptionDataParametersSplat - if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { + $subscriptionScopeKnownErrors = @( + 'Unauthorized', + 'OfferNotSupported', + 'InvalidQueryDefinition', + 'NonValidWebDirectAIRSOfferType', + 'NotFoundNotSupported', + 'IndirectCostDisabled', + 'SubscriptionCostDisabled' + ) + + if ($subConsumptionData -in $subscriptionScopeKnownErrors) { Write-Host " Failed ($subConsumptionData) - Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) @@ -3583,7 +3593,7 @@ function getConsumptionv2 { } else { Write-Host " $($subConsumptionData.properties.rows.Count) Consumption data entries (scope Sub $($subNameToProcess) '$($subIdToProcess)')" - if ($subConsumptionData.Count -gt 0) { + if ($subConsumptionData.properties.rows.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $subConsumptionData -subscriptionQuotaId $subQuotaId } } @@ -3591,7 +3601,7 @@ function getConsumptionv2 { } else { Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" - if ($mgConsumptionData.Count -gt 0) { + if ($mgConsumptionData.properties.rows.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name } } diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index 01d0267b..e05a11d9 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,14 +365,14 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.11', + $ProductVersion = '6.4.12', [string] $GithubRepository = 'aka.ms/AzGovViz', # <--- AzAPICall related parameters #consult the AzAPICall GitHub repository for details aka.ms/AzAPICall [string] - $AzAPICallVersion = '1.2.1', + $AzAPICallVersion = '1.2.3', [switch] $DebugAzAPICall, diff --git a/pwsh/dev/functions/getConsumptionv2.ps1 b/pwsh/dev/functions/getConsumptionv2.ps1 index 7fa0df7c..378952af 100644 --- a/pwsh/dev/functions/getConsumptionv2.ps1 +++ b/pwsh/dev/functions/getConsumptionv2.ps1 @@ -226,7 +226,17 @@ } $subConsumptionData = AzAPICall @subConsumptionDataParametersSplat - if ($subConsumptionData -eq 'Unauthorized' -or $subConsumptionData -eq 'OfferNotSupported' -or $subConsumptionData -eq 'InvalidQueryDefinition' -or $subConsumptionData -eq 'NonValidWebDirectAIRSOfferType' -or $subConsumptionData -eq 'NotFoundNotSupported' -or $subConsumptionData -eq 'IndirectCostDisabled') { + $subscriptionScopeKnownErrors = @( + 'Unauthorized', + 'OfferNotSupported', + 'InvalidQueryDefinition', + 'NonValidWebDirectAIRSOfferType', + 'NotFoundNotSupported', + 'IndirectCostDisabled', + 'SubscriptionCostDisabled' + ) + + if ($subConsumptionData -in $subscriptionScopeKnownErrors) { Write-Host " Failed ($subConsumptionData) - Getting Consumption data scope Sub (Subscription: $($subNameToProcess) '$($subIdToProcess)' QuotaId '$($subQuotaId)')" $hlper = $htAllSubscriptionsFromAPI.($subIdToProcess).subDetails $hlper2 = $htSubscriptionsMgPath.($subIdToProcess) @@ -243,7 +253,7 @@ } else { Write-Host " $($subConsumptionData.properties.rows.Count) Consumption data entries (scope Sub $($subNameToProcess) '$($subIdToProcess)')" - if ($subConsumptionData.Count -gt 0) { + if ($subConsumptionData.properties.rows.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $subConsumptionData -subscriptionQuotaId $subQuotaId } } @@ -251,7 +261,7 @@ } else { Write-Host " #batch$($batchCnt)/$(($subscriptionsBatch | Measure-Object).Count) for $($batch.Group.Count) Subscriptions of QuotaId '$($quotaIdGroup.Name)' returned $($mgConsumptionData.properties.rows.Count) Consumption data entries" - if ($mgConsumptionData.Count -gt 0) { + if ($mgConsumptionData.properties.rows.Count -gt 0) { addToAllConsumptionData -consumptiondataFromAPI $mgConsumptionData -subscriptionQuotaId $quotaIdGroup.Name } } diff --git a/version.json b/version.json index 1b1373f4..7b01e3ad 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.11" + "ProductVersion": "6.4.12" } \ No newline at end of file From ae4970aaca1965010099a4bc5700dcc97574db04 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Mon, 15 Jul 2024 22:39:17 +0200 Subject: [PATCH 19/20] 6.4.12 --- pwsh/AzGovVizParallel.ps1 | 2 +- pwsh/dev/devAzGovVizParallel.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index 1ad15d61..e747c92f 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -649,7 +649,7 @@ Param }, [array] - $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #PayAsYouGo_2014-09-01 + $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/understand-cost-mgt-data#supported-microsoft-azure-offers ) $Error.clear() diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index e05a11d9..ebeb014a 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -649,7 +649,7 @@ Param }, [array] - $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #PayAsYouGo_2014-09-01 + $SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery = @('CSP_2015-05-01') #https://learn.microsoft.com/en-us/azure/cost-management-billing/costs/understand-cost-mgt-data#supported-microsoft-azure-offers ) $Error.clear() From a5342a627dbff79e5c237c49e5c250c859c1da67 Mon Sep 17 00:00:00 2001 From: JulianHayward Date: Thu, 15 Aug 2024 22:50:49 +0200 Subject: [PATCH 20/20] 6.5.0 --- .github/workflows/devskim.yml | 4 ++-- .github/workflows/psScriptAnalyzer.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- README.md | 4 ++-- history.md | 4 ++-- pwsh/AzGovVizParallel.ps1 | 7 ++----- pwsh/dev/devAzGovVizParallel.ps1 | 4 ++-- pwsh/dev/functions/getConsumptionv2.ps1 | 3 --- version.json | 2 +- 9 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.github/workflows/devskim.yml b/.github/workflows/devskim.yml index 917d4030..8ee879fe 100644 --- a/.github/workflows/devskim.yml +++ b/.github/workflows/devskim.yml @@ -7,9 +7,9 @@ name: DevSkim on: push: - branches: [ "developmentJH" ] + branches: [ "master" ] pull_request: - branches: [ "developmentJH" ] + branches: [ "master" ] # schedule: # - cron: '28 13 * * 2' diff --git a/.github/workflows/psScriptAnalyzer.yml b/.github/workflows/psScriptAnalyzer.yml index bd7ec582..e913c26d 100644 --- a/.github/workflows/psScriptAnalyzer.yml +++ b/.github/workflows/psScriptAnalyzer.yml @@ -11,9 +11,9 @@ name: PSScriptAnalyzer on: push: - branches: [ "developmentJH" ] + branches: [ "master" ] pull_request: - branches: [ "developmentJH" ] + branches: [ "master" ] # schedule: # - cron: '25 6 * * 1' diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 61b8d1b7..9b2167b9 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -12,7 +12,7 @@ on: # schedule: # - cron: '21 0 * * 1' push: - branches: [ "developmentJH" ] + branches: [ "master" ] # Declare default permissions as read only. permissions: read-all diff --git a/README.md b/README.md index a708cac1..84216b91 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Gov ## Release history -**Changes** (2024-July-15 / 6.4.12 Minor) +**Changes** (2024-August-15 / 6.5.0 Minor/Patch) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -91,7 +91,7 @@ The [Azure Governance Visualizer accelerator](https://github.com/Azure/Azure-Gov - update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) -- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- update getConsumption (getConsumptionv2): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. - html; update jquery; source tablefilter js - update `.devcontainer/devcontainer.json` - use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.3 (Handle costManagement error `SubscriptionCostDisabled`) diff --git a/history.md b/history.md index ae55fa80..a2db2d00 100644 --- a/history.md +++ b/history.md @@ -4,7 +4,7 @@ ### Azure Governance Visualizer version 6 -**Changes** (2024-July-15 / 6.4.12 Minor) +**Changes** (2024-August-15 / 6.5.0 Minor/Patch) - ALZ policy refresh H2 FY24 (initiatives.json) - [DevSkim](https://github.com/microsoft/DevSkim-Action), [PSScriptAnalyzer](https://github.com/microsoft/psscriptanalyzer-action) and [OpenSSF Scorecard](https://github.com/ossf/scorecard?tab=readme-ov-file#scorecard-github-action) integration @@ -13,7 +13,7 @@ - update GitHub workflows to use azure/login@v2 (previous: azure/login@v1): - [AzGovViz_OIDC.yml](/.github/workflows/AzGovViz_OIDC.yml) - [AzGovViz.yml](/.github/workflows/AzGovViz.yml) -- update getConsumption (experimental for now): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. In order to use this you must update the AzGovVizParallel.ps1 file to use the function `getConsumptionv2` instead of `getConsumption` +- update getConsumption (getConsumptionv2): instead of full Management Group scope costmanagement data retrieval, batch by Subscription quotaId in batches of 100. Failing batches and batches of Subscriptions of quotaId `CSP_2015-05-01` (see param block variable `SubscriptionQuotaIdsThatDoNotSupportCostManagementManagementGroupScopeQuery`) will fallback to get costmanagement data per Subscription. - html; update jquery; source tablefilter js - update `.devcontainer/devcontainer.json` - use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.3 (Handle costManagement error `SubscriptionCostDisabled`) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index e747c92f..4a17f393 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.12', + $ProductVersion = '6.5.0', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -3339,9 +3339,6 @@ function getConsumption { Write-Host "Getting Consumption data duration: $((New-TimeSpan -Start $startConsumptionData -End $endConsumptionData).TotalSeconds) seconds" } function getConsumptionv2 { - #todo: remove - Write-Host '#########-----------------#########' -ForegroundColor DarkMagenta - Write-Host 'Executing getConsumptionv2' -ForegroundColor DarkMagenta $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) @@ -35421,7 +35418,7 @@ if (-not $HierarchyMapOnly) { } if ($azAPICallConf['htParameters'].DoAzureConsumption -eq $true) { - getConsumption + getConsumptionv2 } getOrphanedResources diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index ebeb014a..3353a6ab 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -365,7 +365,7 @@ Param $Product = 'AzGovViz', [string] - $ProductVersion = '6.4.12', + $ProductVersion = '6.5.0', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -1123,7 +1123,7 @@ if (-not $HierarchyMapOnly) { } if ($azAPICallConf['htParameters'].DoAzureConsumption -eq $true) { - getConsumption + getConsumptionv2 } getOrphanedResources diff --git a/pwsh/dev/functions/getConsumptionv2.ps1 b/pwsh/dev/functions/getConsumptionv2.ps1 index 378952af..a0eee847 100644 --- a/pwsh/dev/functions/getConsumptionv2.ps1 +++ b/pwsh/dev/functions/getConsumptionv2.ps1 @@ -1,7 +1,4 @@ function getConsumptionv2 { - #todo: remove - Write-Host '#########-----------------#########' -ForegroundColor DarkMagenta - Write-Host 'Executing getConsumptionv2' -ForegroundColor DarkMagenta $costManagementQueryAPIVersion = $azAPICallConf['htParameters'].APIMappingCloudEnvironment.costManagementQuery.($azAPICallConf['htParameters'].azureCloudEnvironment) diff --git a/version.json b/version.json index 7b01e3ad..1c9599c2 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "ProductVersion": "6.4.12" + "ProductVersion": "6.5.0" } \ No newline at end of file