diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json index d40425bf3a..020a7fad87 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*Lua*", @@ -13,14 +10,36 @@ "expectedBasicLuaRunner2MeanValue_win": 173, "expectedBasicLuaRunner3MeanValue_win": 686, "expectedBasicLua4RunnerMeanValue_win": 2711, + + "expectedBasicLuaRunner1AllocatedValue_linux": 24, + "expectedBasicLuaRunner2AllocatedValue_linux": 144, + "expectedBasicLuaRunner3AllocatedValue_linux": 328, + "expectedBasicLua4RunnerAllocatedValue_linux": 864, + "expectedBasicLuaRunner1AllocatedValue_win": 24, + "expectedBasicLuaRunner2AllocatedValue_win": 144, + "expectedBasicLuaRunner3AllocatedValue_win": 328, + "expectedBasicLua4RunnerAllocatedValue_win": 864, + + "expectedBasicLua1MeanValue_linux": 169, "expectedBasicLua2MeanValue_linux": 284, "expectedBasicLua3MeanValue_linux": 1407, "expectedBasicLua4MeanValue_linux": 2733, + "expectedBasicLua1AllocatedValue_linux": 24, + "expectedBasicLua2AllocatedValue_linux": 144, + "expectedBasicLua3AllocatedValue_linux": 328, + "expectedBasicLua4AllocatedValue_linux": 832, + "expectedBasicLua1MeanValue_win": 107, "expectedBasicLua2MeanValue_win": 180, "expectedBasicLua3MeanValue_win": 660, "expectedBasicLua4MeanValue_win": 1805, + "expectedBasicLua1AllocatedValue_win": 24, + "expectedBasicLua2AllocatedValue_win": 144, + "expectedBasicLua3AllocatedValue_win": 328, + "expectedBasicLua4AllocatedValue_win": 832, + + "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json index 4b38979781..8692bff1b9 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterBench*", diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json index 5f4387832b..620bb8c509 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterMigrateBench*", diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json index 0995827e35..0ef19f2aa4 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespParseStress*", @@ -14,6 +11,14 @@ "expectedSAddRemMeanValue_linux": 172, "expectedHSetDelMeanValue_linux": 200, "expectedMyDictSetGetMeanValue_linux": 235, + + "expectedZAddRemAllocatedValue_linux": 23552, + "expectedLPushPopAllocatedValue_linux": 30721, + "expectedSAddRemAllocatedValue_linux": 16384, + "expectedHSetDelAllocatedValue_linux": 55297, + "expectedMyDictSetGetAllocatedValue_linux": 30720, + + "expectedInLinePingMeanValue_win": 2.4, "expectedSETMeanValue_win": 19, "expectedSETEXMeanValue_win": 26, @@ -23,6 +28,13 @@ "expectedSAddRemMeanValue_win": 130, "expectedHSetDelMeanValue_win": 160, "expectedMyDictSetGetMeanValue_win": 242, + + "expectedZAddRemAllocatedValue_win": 23552, + "expectedLPushPopAllocatedValue_win": 30721, + "expectedSAddRemAllocatedValue_win": 16384, + "expectedHSetDelAllocatedValue_win": 55297, + "expectedMyDictSetGetAllocatedValue_win": 30720, + "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json index ff0c0333c1..c7a1cfe0a0 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespTsavoriteStress*", diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index c424ab988d..76f3d5964b 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -42,17 +42,17 @@ function AnalyzeResult { # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Mean Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Mean Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Mean Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } return $false # the values are too different @@ -76,6 +76,7 @@ param ($ResultsLine, $columnNum) $column = $columns | ForEach-Object { $_.Trim() } $foundValue = $column[$columnNum].Trim(' us') $foundValue = $foundValue.Trim(' ns') + $foundValue = $foundValue.Trim(' B') return $foundValue }