Skip to content

Commit

Permalink
Add results from parameterized tests to parent (#16)
Browse files Browse the repository at this point in the history 
This update will ensure that the correct test case gets updated when a
parameterized test is run:
- When test case results are sent back from the build server, a failure
will be sent for each parameterized test case.
- Extract the original test case name (language-specific), which will be
used to lookup the correct parent item
- Each entry, which contains the specific failure details for that
entry, will now get added to the parent message. This will surface the
specific cases in the parameterized test that failed.
  Example:

![image](https://github.com/uber/vscode-bazel-bsp/assets/92764374/2cab4488-1d15-4fa3-8e81-8acfa37b25c2)
  • Loading branch information
@mnoah1
mnoah1 authored Jul 2, 2024
1 parent 2dca280 commit ba94d71
Show file tree
Hide file tree
Showing 6 changed files with 554 additions and 57 deletions.
19 changes: 17 additions & 2 deletions src/language-tools/java.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const JAVA_TEST_REGEX =
/@Test\s+.*\s+public void (?<methodName>\w+)|public class (?<className>(Test\w*|\w+Test))\s+extends/
const PACKAGE_NAME_REGEX =
/package\s+(?<packageName>([a-zA-Z_][a-zA-Z0-9_]*)(\.[a-zA-Z_][a-zA-Z0-9_]*)*);/
const PARAMETERIZED_TEST_REGEX = /^(?<lookupKey>.*?)(?=\[.*?\])(.*)$/

export class JavaLanguageTools implements LanguageTools {
/**
Expand All @@ -19,10 +20,24 @@ export class JavaLanguageTools implements LanguageTools {
* @returns Lookup key to find this test case in the TestRunTracker.
*/
mapTestFinishDataToLookupKey(testFinishData: TestFinish): string | undefined {
if (testFinishData.dataKind === TestFinishDataKind.JUnitStyleTestCaseData) {
if (
testFinishData.dataKind === TestFinishDataKind.JUnitStyleTestCaseData &&
testFinishData.data
) {
const testCaseData = testFinishData.data as JUnitStyleTestCaseData
if (testCaseData.className !== undefined) {
return `${testCaseData.className}.${testFinishData.displayName}`
let testCaseName = testFinishData.displayName

// In case of a parameterized test, keep the method name.
const match = testCaseName.match(PARAMETERIZED_TEST_REGEX)
if (match?.groups?.lookupKey) {
testCaseName = match.groups.lookupKey
}

// Use the class name as the base, and append the test case name if available.
let result = testCaseData.className
if (testCaseName.length > 0) result += `.${testCaseName}`
return result
} else {
return testFinishData.displayName
}
Expand Down
19 changes: 17 additions & 2 deletions src/language-tools/python.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {BaseLanguageTools} from './base'
import {JUnitStyleTestCaseData, TestFinishDataKind} from '../bsp/bsp-ext'

const TEST_FILE_REGEX = /^(test_.+\.py|.+_test\.py)$/
const PARAMETERIZED_TEST_REGEX = /^(?<lookupKey>.*?)(?=\[.*?\])(.*)$/

export class PythonLanguageTools
extends BaseLanguageTools
Expand All @@ -21,9 +22,23 @@ export class PythonLanguageTools
* @returns Lookup key to find this test case in the TestRunTracker.
*/
mapTestFinishDataToLookupKey(testFinishData: TestFinish): string | undefined {
if (testFinishData.dataKind === TestFinishDataKind.JUnitStyleTestCaseData) {
if (
testFinishData.dataKind === TestFinishDataKind.JUnitStyleTestCaseData &&
testFinishData.data
) {
const testCaseData = testFinishData.data as JUnitStyleTestCaseData
return `${testCaseData.className}.${testFinishData.displayName}`
let testCaseName = testFinishData.displayName

// In case of a parameterized test, keep the method name.
const match = testCaseName.match(PARAMETERIZED_TEST_REGEX)
if (match?.groups?.lookupKey) {
testCaseName = match.groups.lookupKey
}

// Use the class name as the base, and append the test case name if available.
let result = testCaseData.className
if (testCaseName.length > 0) result += `.${testCaseName}`
return result
}
return undefined
}
Expand Down
15 changes: 14 additions & 1 deletion src/test-runner/run-tracker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ export class TestRunTracker implements TaskOriginHandlers {

/**
* Updates an item's status, both within this test run tracker and in the test explorer UI.
* This may be called more than once for the same test item, in which case only the highest status (per TestCaseStatus enum) will be retained.
* @param item TestItem for which the status will be updated.
* @param status New status value.
* @param message (optional) Message to be shown to report an outcome. Only applicable for Failed and Errored states.
Expand All @@ -137,6 +138,13 @@ export class TestRunTracker implements TaskOriginHandlers {
status: TestCaseStatus,
message?: vscode.TestMessage
) {
const currentStatus = this.status.get(item)
if (currentStatus && status < currentStatus) {
// Only update if the new status is ranked higher than the existing one.
// This allows multiple updates to be made to a test item, while only showing the highest status in the UI.
return
}

this.status.set(item, status)
switch (status) {
case TestCaseStatus.Started:
Expand Down Expand Up @@ -395,7 +403,9 @@ export class TestRunTracker implements TaskOriginHandlers {
}
}

function formatTestResultMessage(result): vscode.TestMessage | undefined {
function formatTestResultMessage(
result: TestFinish
): vscode.TestMessage | undefined {
let message =
// Ignore 'null' string as well.
// TODO(IDE-1133): Ensure server does not convert null values to string.
Expand All @@ -405,6 +415,9 @@ function formatTestResultMessage(result): vscode.TestMessage | undefined {

if (result.dataKind === TestFinishDataKind.JUnitStyleTestCaseData) {
const testCaseData = result.data as JUnitStyleTestCaseData
if (result.displayName) {
message += `${ANSI_CODES.RED}[TEST CASE]${ANSI_CODES.RESET} ${result.displayName}\n\n`
}
if (testCaseData.errorType && testCaseData.fullError !== 'null') {
message += `${ANSI_CODES.RED}[ERROR TYPE]${ANSI_CODES.RESET} ${testCaseData.errorType}\n\n`
}
Expand Down
266 changes: 242 additions & 24 deletions src/test/suite/language-tools/java.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,34 +99,252 @@ suite('Java Language Tools', () => {
assert.strictEqual(result.testCases.length, 0)
})

test('map test finish data to lookup key', async () => {
let result = languageTools.mapTestFinishDataToLookupKey({
displayName: 'myTest',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
const testCases = [
{
description: 'test method within a class',
input: {
displayName: 'myTest',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
},
},
})
assert.strictEqual(result, 'com.example.ClassName.myTest')

result = languageTools.mapTestFinishDataToLookupKey({
displayName: 'com.example.MySuite',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
expected: 'com.example.ClassName.myTest',
},
{
description: 'suite level test case',
input: {
displayName: 'com.example.MySuite',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
},
},
})
assert.strictEqual(result, 'com.example.MySuite')
expected: 'com.example.MySuite',
},
{
description: 'no dataKind provided',
input: {
displayName: 'com.example.MySuite',
status: TestStatus.Failed,
},
expected: undefined,
},
{
description: 'parameterized test cases',
input: {
displayName: 'myTest[example1]',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
},
},
expected: 'com.example.ClassName.myTest',
},
{
description: 'parameterized test with special characters',
input: {
displayName: 'testMethod[example1!@#]',
status: TestStatus.Passed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 1,
className: 'com.example.SpecialCharsExample',
},
},
expected: 'com.example.SpecialCharsExample.testMethod',
},
{
description: 'parameterized test with spaces',
input: {
displayName: 'testMethod[example with spaces]',
status: TestStatus.Skipped,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0.5,
className: 'com.example.SpaceTestExample',
},
},
expected: 'com.example.SpaceTestExample.testMethod',
},
{
description: 'parameterized test with multiple brackets',
input: {
displayName: 'testMethod[example[inner]]',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 2,
className: 'com.example.MultiBracketTestExample',
},
},
expected: 'com.example.MultiBracketTestExample.testMethod',
},
{
description: 'parameterized test with numbers',
input: {
displayName: 'testMethod[12345]',
status: TestStatus.Passed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0.1,
className: 'com.example.NumericTestExample',
},
},
expected: 'com.example.NumericTestExample.testMethod',
},
{
description: 'parameterized test with empty brackets',
input: {
displayName: 'testMethod[]',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 1.5,
className: 'com.example.EmptyBracketTestExample',
},
},
expected: 'com.example.EmptyBracketTestExample.testMethod',
},
{
description: 'parameterized test with special symbols',
input: {
displayName: 'testMethod[!@#$%^&*()]',
status: TestStatus.Skipped,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 3,
className: 'com.example.SymbolsTestExample',
},
},
expected: 'com.example.SymbolsTestExample.testMethod',
},
{
description: 'parameterized test with long name',
input: {
displayName: 'testMethod[averylongsubtestnamethatisunusuallylong]',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 2.5,
className: 'com.example.LongNameTestExample',
},
},
expected: 'com.example.LongNameTestExample.testMethod',
},
{
description: 'parameterized test with nested brackets',
input: {
displayName: 'testMethod[example[nested[brackets]]]',
status: TestStatus.Passed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0.2,
className: 'com.example.NestedBracketsTestExample',
},
},
expected: 'com.example.NestedBracketsTestExample.testMethod',
},
{
description: 'successful tests with data',
input: {
displayName: 'mySuccessfulTest',
status: TestStatus.Passed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 1,
className: 'com.example.SuccessClass',
},
},
expected: 'com.example.SuccessClass.mySuccessfulTest',
},
{
description: 'tests with no className',
input: {
displayName: 'myTestWithoutClass',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 2,
},
},
expected: 'myTestWithoutClass',
},
{
description: 'unknown dataKind',
input: {
displayName: 'unknownTest',
status: TestStatus.Failed,
dataKind: 'UnknownDataKind',
data: {
time: 0,
className: 'com.example.UnknownClass',
},
},
expected: undefined,
},
{
description: 'null data gracefully',
input: {
displayName: 'nullDataTest',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: null,
},
expected: undefined,
},
{
description: 'numeric displayName',
input: {
displayName: '123456',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
},
},
expected: 'com.example.ClassName.123456',
},
{
description: 'special characters in displayName',
input: {
displayName: '!@#$%^&*()',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
},
},
expected: 'com.example.ClassName.!@#$%^&*()',
},
{
description: 'empty string as displayName',
input: {
displayName: '',
status: TestStatus.Failed,
dataKind: TestFinishDataKind.JUnitStyleTestCaseData,
data: {
time: 0,
className: 'com.example.ClassName',
},
},
expected: 'com.example.ClassName',
},
]

result = languageTools.mapTestFinishDataToLookupKey({
displayName: 'com.example.MySuite',
status: TestStatus.Failed,
for (const testCase of testCases) {
test(`map test finish data to lookup key: ${testCase.description}`, async () => {
const result = languageTools.mapTestFinishDataToLookupKey(testCase.input)
assert.strictEqual(result, testCase.expected)
})
assert.strictEqual(result, undefined)
})
}

test('map test case info to lookup key', async () => {
let testInfo = testController.createTestItem('test1', 'test1')
Expand Down
Loading

0 comments on commit ba94d71

Please sign in to comment.