diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09b2b1d4..e240774c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,26 +4,66 @@ name: Node.js CI on: - push: - branches: [ "develop" ] - pull_request: - branches: [ "develop" ] + push: + branches: ["develop"] + pull_request: + branches: ["develop"] jobs: - build: - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: 18.x - - run: npm install - - run: xvfb-run -a npm test - if: runner.os == 'Linux' - - run: npm test - if: runner.os != 'Linux' + test: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 18.x + + - run: npm install + + - name: Run Tests + run: xvfb-run -a npm test + if: runner.os == 'Linux' + + - name: Run Tests (Non-Linux) + run: npm test + if: runner.os != 'Linux' + + stability-test: + needs: test # Runs only if 'test' job is successful + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 18.x + + - run: npm install + + - name: Compile + run: xvfb-run -a npm run compile + if: runner.os == 'Linux' + + - name: Get ADE Repo + run: xvfb-run -a npm run get-ade-test + if: runner.os == 'Linux' + + - name: Run Stability Test + run: xvfb-run -a npm run stability-test + if: runner.os == 'Linux' + + - name: Save Stability Test Results as Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: results + path: /home/runner/work/AblFormatter/AblFormatter/resources/testResults + retention-days: 7 diff --git a/.vscode/settings.json b/.vscode/settings.json index d16a58ae..5c3e53c1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,4 +9,6 @@ "[typescript]" : { "editor.defaultFormatter": "esbenp.prettier-vscode" }, + + "editor.formatOnSave": true } \ No newline at end of file diff --git a/package.json b/package.json index 77e96880..b17ce72b 100644 --- a/package.json +++ b/package.json @@ -327,7 +327,8 @@ "copy-test-cases": "copyfiles -u 2 ./resources/samples/* .test_dir/samples", "copy-test-settings": "copyfiles -u 2 ./resources/samples/.vscode/* .test_dir/samples/", "test": "node ./out/test/runTest.js", - "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade" + "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade", + "stability-test": "node ./out/stability-test/runTest.js" }, "devDependencies": { "@electron/rebuild": "^3.3.1", diff --git a/resources/stabilityTests/_failures.txt b/resources/stabilityTests/_failures.txt new file mode 100644 index 00000000..8be4e527 --- /dev/null +++ b/resources/stabilityTests/_failures.txt @@ -0,0 +1,143 @@ +adecomm_aud-tts.i +adecomm_dswid.i +adecomm_peditor.i +adecomm_xmlschema.p +adecomm_xmlwidg.i +adecomm__dswidfunc.p +adecomm__pweditor.i +adecomm__reg2ini.p +adedict_FLD__dfltgat.p +adedict_FLD__fldgate.p +adedict_IDX__newidx.p +adeedit__winrsz.p +aderes_aderes.p +aderes_j-table2.p +aderes_l-verify.p +aderes_s-exp.p +aderesc__read.p +adeshar__cntrapi.p +adeshar__mchgi.p +adeuib_ide_request__requestmanager.cls +adeuib_sookver.i +adeuib_ttobject.i +adeuib_vrfyimp.i +adeuib__cuelist.w +adeuib__edtmenu.p +adeuib__genmult.p +adeuib__rdproc.p +adeuib__rsz_wp.p +adeuib__seresz.p +adeuib__ttyedit.p +adeuib__ttyss.i +adeuib__tview.w +adeweb_rdproc.i +adeweb__rddefs.p +adexml__xmlview.w +adm2_support_filterd.w +adm2_support_folderd.w +adm2_widgetprto.i +auditing_include__xmlsec.i +auditing_sdo__audfieldpolicysdo.w +auditing_sdo__audfilepolicysdo.w +auditing_src_adm2_widgetprto.i +auditing_ttdefs__audeventpolicytt.i +auditing_ttdefs__audeventtt.i +auditing_ttdefs__audfieldpolicytt.i +auditing_ttdefs__audfilepolicytt.i +auditing_ttdefs__audpolicytt.i +corelib_OpenEdge_Core_Collections_AbstractTTCollection.cls +netlib_OpenEdge_Web_DataObject_Writer_FieldElementWriter.cls +OpenEdge_DataAdmin_Binding_CreateContext.cls +OpenEdge_DataAdmin_Binding_Query_AreaExtentQuery.cls +OpenEdge_DataAdmin_Binding_Query_SchemaPartitionQuery.cls +OpenEdge_DataAdmin_Binding_SequenceValueContext.cls +OpenEdge_DataAdmin_Core_JSONLoader.cls +OpenEdge_DataAdmin_DataAccess_AllFileData.cls +OpenEdge_DataAdmin_DataFileList.cls +OpenEdge_DataAdmin_DataSource_IndexDataSource.cls +OpenEdge_DataAdmin_DataSource_UserPermissionDataSource.cls +OpenEdge_DataAdmin_ExtentList.cls +OpenEdge_DataAdmin_IndexFieldList.cls +OpenEdge_DataAdmin_IPartitionCollection.cls +OpenEdge_DataAdmin_ITable.cls +OpenEdge_DataAdmin_IUserTablePermission.cls +OpenEdge_DataAdmin_Lang_QueryString.cls +OpenEdge_DataAdmin_Message_IFetchRequest.cls +OpenEdge_DataAdmin_Message_ISaveRequest.cls +OpenEdge_DataAdmin_Message_ITableRequest.cls +OpenEdge_DataAdmin_PartitionPolicyField.cls +OpenEdge_DataAdmin_Rest_RestRequest.cls +OpenEdge_DataAdmin_ServerCommand_ProstrctAddOnlineCommand.cls +OpenEdge_DataAdmin_TenantTypes.cls +OpenEdge_DataAdmin_Util_CDCTrackingHelper.cls +OpenEdge_DataAdmin_Util_dbCanMigrate.p +OpenEdge_DataAdmin_Util_ITableImportUtility.cls +prodict_dictsplt.i +prodict_dictsrch.i +prodict_dump__dmpincr.p +prodict_dump__dmpsec.p +prodict_dump__dmputil.p +prodict_dump__lodsddl.p +prodict_dump__lodsec.p +prodict_gate_gatework.i +prodict_gate__gat_cro.p +prodict_gate__gat_nam.p +prodict_mss__mss_pul.p +prodict_odb__odb_pul.p +prodict_ora__ora_typ.p +prodict_pro__pro_fld.p +prodict_sec_sec-pol.i +prodict_user__usrtget.p +prohelp_keyword.p +prores_a-fast.p +prores_a-write.p +prores_c-edit.i +prores_l-guess.p +prores_l-verify.p +prores_l-write.p +prores_q-join.p +prores_r-ft.p +prores_r-main.p +prores_r-write.i +prores_reslang_t-a-dan.p +prores_reslang_t-a-fr7.p +prores_reslang_t-a-fre.p +prores_reslang_t-b-fr7.p +prores_reslang_t-b-fre.p +prores_reslang_t-c-fr7.p +prores_reslang_t-c-fre.p +prores_reslang_t-d-fr7.p +prores_reslang_t-d-fre.p +prores_reslang_t-i-eng.p +prores_reslang_t-i-fr7.p +prores_reslang_t-i-fre.p +prores_reslang_t-l-fr7.p +prores_reslang_t-l-fre.p +prores_reslang_t-l-gr1.p +prores_reslang_t-l-gre.p +prores_reslang_t-q-fr7.p +prores_reslang_t-q-fre.p +prores_reslang_t-q-gr1.p +prores_reslang_t-q-gre.p +prores_reslang_t-r-fr7.p +prores_reslang_t-r-fre.p +prores_reslang_t-s-eng.p +prores_reslang_t-s-fr7.p +prores_reslang_t-s-fre.p +prores_s-gen2.i +prores_s-info.p +prores_s-order.p +prores_t-i-eng.p +prores_t-s-eng.p +prores_u-export.p +prores_u-load.p +protools__dblist.w +protools__scrcap.w +protools__session.w +protools__v89conv.w +web_examples_w-custdir.w +webtools_util__dirlist.w +workshop_dirnode.i +workshop_rdproc.i +workshop__methlib.w +workshop__rdproc.p diff --git a/src/stability-test/suite/extension.test.ts b/src/stability-test/suite/extension.test.ts index b9f5b530..ff6d1448 100644 --- a/src/stability-test/suite/extension.test.ts +++ b/src/stability-test/suite/extension.test.ts @@ -34,6 +34,21 @@ const testRunTimestamp = new Date() .substring(0, 19); const testRunDir = join(testResultsDir, testRunTimestamp); +const knownFailures = getFailedTestCases( + join(extensionDevelopmentPath, "resources/stabilityTests") +); +const settingsOverride = + "/* formatterSettingsOverride */\n/*" + + readFile( + join( + extensionDevelopmentPath, + "resources/stabilityTests/.vscode/settings.json" + ) + ) + + "*/\n"; + +console.log(settingsOverride); + suite("Extension Test Suite", () => { console.log("Parser initialized", stabilityTestCases); @@ -74,13 +89,25 @@ function stabilityTest(name: string): void { ConfigurationManager2.getInstance(); enableFormatterDecorators(); - const beforeText = getInput(name); + const beforeText = settingsOverride + getInput(name); const beforeCount = countActualSymbols(beforeText); const afterText = format(beforeText, name); const afterCount = countActualSymbols(afterText); + const nameWithRelativePath = name.startsWith(stabilityTestDir) + ? name.slice(stabilityTestDir.length + 1) + : name; + + const fileName = nameWithRelativePath.replace(/[\s\/\\:*?"<>|]+/g, "_"); + if (beforeCount !== afterCount) { - const fileName = path.basename(name, path.extname(name)); + if (knownFailures.includes(fileName)) { + console.log("Known issue"); + return; + } + + addFailedTestCase(testRunDir, "_failures.txt", fileName); + const beforeFilePath = join( testRunDir, `${fileName}_before${path.extname(name)}` @@ -98,7 +125,13 @@ function stabilityTest(name: string): void { After: ${afterFilePath} `); } - // assert.strictEqual(beforeCount, afterCount); + + // if test passes but file is in error list + if (knownFailures.includes(fileName)) { + addFailedTestCase(testRunDir, "_new_passes.txt", fileName); + + assert.fail(`File should fail ${fileName}`); + } } function getInput(fileName: string): string { @@ -241,3 +274,32 @@ function formatErrorMessage(errors: Parser.SyntaxNode[], name: string): string { return errorMessage; } + +function getFailedTestCases(filePath: string): string[] { + const failedFilePath = path.join(filePath, "_failures.txt"); + + // Check if the file exists to avoid errors + if (!fs.existsSync(failedFilePath)) { + console.log("Known Failures file does not exist!"); + return []; + } + + // Read the file and split lines into an array + const data = fs.readFileSync(failedFilePath, "utf8"); + const failures = data.split("\n").filter((line) => line.trim() !== ""); + + console.log("Known failures list has ", failures.length, "cases"); + + return failures; +} + +function addFailedTestCase( + filePath: string, + fileName: string, + failedCase: string +): void { + const failedFilePath = path.join(filePath, fileName); + + // Append the failed test case to the file with a newline + fs.appendFileSync(failedFilePath, failedCase + "\n", "utf8"); +}