Merge pull request #302 from BalticAmadeus/301-aed-testing-update

301 aed testing update

.github/workflows/main.yml

@@ -4,26 +4,66 @@
 name: Node.js CI
 
 on:
-  push:
-    branches: [ "develop" ]
-  pull_request:
-    branches: [ "develop" ]
+    push:
+        branches: ["develop"]
+    pull_request:
+        branches: ["develop"]
 
 jobs:
-  build:
-    strategy:
-      matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Install Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: 18.x
-    - run: npm install
-    - run: xvfb-run -a npm test
-      if: runner.os == 'Linux'
-    - run: npm test
-      if: runner.os != 'Linux'
+    test:
+        strategy:
+            matrix:
+                os: [ubuntu-latest, windows-latest, macos-latest]
+        runs-on: ${{ matrix.os }}
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Install Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18.x
+
+            - run: npm install
+
+            - name: Run Tests
+              run: xvfb-run -a npm test
+              if: runner.os == 'Linux'
+
+            - name: Run Tests (Non-Linux)
+              run: npm test
+              if: runner.os != 'Linux'
+
+    stability-test:
+        needs: test # Runs only if 'test' job is successful
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Install Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18.x
+
+            - run: npm install
+
+            - name: Compile
+              run: xvfb-run -a npm run compile
+              if: runner.os == 'Linux'
+
+            - name: Get ADE Repo
+              run: xvfb-run -a npm run get-ade-test
+              if: runner.os == 'Linux'
+
+            - name: Run Stability Test
+              run: xvfb-run -a npm run stability-test
+              if: runner.os == 'Linux'
+
+            - name: Save Stability Test Results as Artifact
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: results
+                  path: /home/runner/work/AblFormatter/AblFormatter/resources/testResults
+                  retention-days: 7

.vscode/settings.json

@@ -9,4 +9,6 @@
      "[typescript]" : {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
      },
+
+     "editor.formatOnSave": true
 }

package.json

@@ -327,7 +327,8 @@
     "copy-test-cases": "copyfiles -u 2 ./resources/samples/* .test_dir/samples",
     "copy-test-settings": "copyfiles -u 2 ./resources/samples/.vscode/* .test_dir/samples/",
     "test": "node ./out/test/runTest.js",
-    "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade"
+    "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade",
+    "stability-test": "node ./out/stability-test/runTest.js"
   },
   "devDependencies": {
     "@electron/rebuild": "^3.3.1",

resources/stabilityTests/_failures.txt

@@ -0,0 +1,143 @@
+adecomm_aud-tts.i
+adecomm_dswid.i
+adecomm_peditor.i
+adecomm_xmlschema.p
+adecomm_xmlwidg.i
+adecomm__dswidfunc.p
+adecomm__pweditor.i
+adecomm__reg2ini.p
+adedict_FLD__dfltgat.p
+adedict_FLD__fldgate.p
+adedict_IDX__newidx.p
+adeedit__winrsz.p
+aderes_aderes.p
+aderes_j-table2.p
+aderes_l-verify.p
+aderes_s-exp.p
+aderesc__read.p
+adeshar__cntrapi.p
+adeshar__mchgi.p
+adeuib_ide_request__requestmanager.cls
+adeuib_sookver.i
+adeuib_ttobject.i
+adeuib_vrfyimp.i
+adeuib__cuelist.w
+adeuib__edtmenu.p
+adeuib__genmult.p
+adeuib__rdproc.p
+adeuib__rsz_wp.p
+adeuib__seresz.p
+adeuib__ttyedit.p
+adeuib__ttyss.i
+adeuib__tview.w
+adeweb_rdproc.i
+adeweb__rddefs.p
+adexml__xmlview.w
+adm2_support_filterd.w
+adm2_support_folderd.w
+adm2_widgetprto.i
+auditing_include__xmlsec.i
+auditing_sdo__audfieldpolicysdo.w
+auditing_sdo__audfilepolicysdo.w
+auditing_src_adm2_widgetprto.i
+auditing_ttdefs__audeventpolicytt.i
+auditing_ttdefs__audeventtt.i
+auditing_ttdefs__audfieldpolicytt.i
+auditing_ttdefs__audfilepolicytt.i
+auditing_ttdefs__audpolicytt.i
+corelib_OpenEdge_Core_Collections_AbstractTTCollection.cls
+netlib_OpenEdge_Web_DataObject_Writer_FieldElementWriter.cls
+OpenEdge_DataAdmin_Binding_CreateContext.cls
+OpenEdge_DataAdmin_Binding_Query_AreaExtentQuery.cls
+OpenEdge_DataAdmin_Binding_Query_SchemaPartitionQuery.cls
+OpenEdge_DataAdmin_Binding_SequenceValueContext.cls
+OpenEdge_DataAdmin_Core_JSONLoader.cls
+OpenEdge_DataAdmin_DataAccess_AllFileData.cls
+OpenEdge_DataAdmin_DataFileList.cls
+OpenEdge_DataAdmin_DataSource_IndexDataSource.cls
+OpenEdge_DataAdmin_DataSource_UserPermissionDataSource.cls
+OpenEdge_DataAdmin_ExtentList.cls
+OpenEdge_DataAdmin_IndexFieldList.cls
+OpenEdge_DataAdmin_IPartitionCollection.cls
+OpenEdge_DataAdmin_ITable.cls
+OpenEdge_DataAdmin_IUserTablePermission.cls
+OpenEdge_DataAdmin_Lang_QueryString.cls
+OpenEdge_DataAdmin_Message_IFetchRequest.cls
+OpenEdge_DataAdmin_Message_ISaveRequest.cls
+OpenEdge_DataAdmin_Message_ITableRequest.cls
+OpenEdge_DataAdmin_PartitionPolicyField.cls
+OpenEdge_DataAdmin_Rest_RestRequest.cls
+OpenEdge_DataAdmin_ServerCommand_ProstrctAddOnlineCommand.cls
+OpenEdge_DataAdmin_TenantTypes.cls
+OpenEdge_DataAdmin_Util_CDCTrackingHelper.cls
+OpenEdge_DataAdmin_Util_dbCanMigrate.p
+OpenEdge_DataAdmin_Util_ITableImportUtility.cls
+prodict_dictsplt.i
+prodict_dictsrch.i
+prodict_dump__dmpincr.p
+prodict_dump__dmpsec.p
+prodict_dump__dmputil.p
+prodict_dump__lodsddl.p
+prodict_dump__lodsec.p
+prodict_gate_gatework.i
+prodict_gate__gat_cro.p
+prodict_gate__gat_nam.p
+prodict_mss__mss_pul.p
+prodict_odb__odb_pul.p
+prodict_ora__ora_typ.p
+prodict_pro__pro_fld.p
+prodict_sec_sec-pol.i
+prodict_user__usrtget.p
+prohelp_keyword.p
+prores_a-fast.p
+prores_a-write.p
+prores_c-edit.i
+prores_l-guess.p
+prores_l-verify.p
+prores_l-write.p
+prores_q-join.p
+prores_r-ft.p
+prores_r-main.p
+prores_r-write.i
+prores_reslang_t-a-dan.p
+prores_reslang_t-a-fr7.p
+prores_reslang_t-a-fre.p
+prores_reslang_t-b-fr7.p
+prores_reslang_t-b-fre.p
+prores_reslang_t-c-fr7.p
+prores_reslang_t-c-fre.p
+prores_reslang_t-d-fr7.p
+prores_reslang_t-d-fre.p
+prores_reslang_t-i-eng.p
+prores_reslang_t-i-fr7.p
+prores_reslang_t-i-fre.p
+prores_reslang_t-l-fr7.p
+prores_reslang_t-l-fre.p
+prores_reslang_t-l-gr1.p
+prores_reslang_t-l-gre.p
+prores_reslang_t-q-fr7.p
+prores_reslang_t-q-fre.p
+prores_reslang_t-q-gr1.p
+prores_reslang_t-q-gre.p
+prores_reslang_t-r-fr7.p
+prores_reslang_t-r-fre.p
+prores_reslang_t-s-eng.p
+prores_reslang_t-s-fr7.p
+prores_reslang_t-s-fre.p
+prores_s-gen2.i
+prores_s-info.p
+prores_s-order.p
+prores_t-i-eng.p
+prores_t-s-eng.p
+prores_u-export.p
+prores_u-load.p
+protools__dblist.w
+protools__scrcap.w
+protools__session.w
+protools__v89conv.w
+web_examples_w-custdir.w
+webtools_util__dirlist.w
+workshop_dirnode.i
+workshop_rdproc.i
+workshop__methlib.w
+workshop__rdproc.p

src/stability-test/suite/extension.test.ts

@@ -34,6 +34,21 @@ const testRunTimestamp = new Date()
     .substring(0, 19);
 const testRunDir = join(testResultsDir, testRunTimestamp);
 
+const knownFailures = getFailedTestCases(
+    join(extensionDevelopmentPath, "resources/stabilityTests")
+);
+const settingsOverride =
+    "/* formatterSettingsOverride */\n/*" +
+    readFile(
+        join(
+            extensionDevelopmentPath,
+            "resources/stabilityTests/.vscode/settings.json"
+        )
+    ) +
+    "*/\n";
+
+console.log(settingsOverride);
+
 suite("Extension Test Suite", () => {
     console.log("Parser initialized", stabilityTestCases);
 
@@ -74,13 +89,25 @@ function stabilityTest(name: string): void {
     ConfigurationManager2.getInstance();
     enableFormatterDecorators();
 
-    const beforeText = getInput(name);
+    const beforeText = settingsOverride + getInput(name);
     const beforeCount = countActualSymbols(beforeText);
     const afterText = format(beforeText, name);
     const afterCount = countActualSymbols(afterText);
 
+    const nameWithRelativePath = name.startsWith(stabilityTestDir)
+        ? name.slice(stabilityTestDir.length + 1)
+        : name;
+
+    const fileName = nameWithRelativePath.replace(/[\s\/\\:*?"<>|]+/g, "_");
+
     if (beforeCount !== afterCount) {
-        const fileName = path.basename(name, path.extname(name));
+        if (knownFailures.includes(fileName)) {
+            console.log("Known issue");
+            return;
+        }
+
+        addFailedTestCase(testRunDir, "_failures.txt", fileName);
+
         const beforeFilePath = join(
             testRunDir,
             `${fileName}_before${path.extname(name)}`
@@ -98,7 +125,13 @@ function stabilityTest(name: string): void {
         After: ${afterFilePath}
         `);
     }
-    // assert.strictEqual(beforeCount, afterCount);
+
+    // if test passes but file is in error list
+    if (knownFailures.includes(fileName)) {
+        addFailedTestCase(testRunDir, "_new_passes.txt", fileName);
+
+        assert.fail(`File should fail ${fileName}`);
+    }
 }
 
 function getInput(fileName: string): string {
@@ -241,3 +274,32 @@ function formatErrorMessage(errors: Parser.SyntaxNode[], name: string): string {
 
     return errorMessage;
 }
+
+function getFailedTestCases(filePath: string): string[] {
+    const failedFilePath = path.join(filePath, "_failures.txt");
+
+    // Check if the file exists to avoid errors
+    if (!fs.existsSync(failedFilePath)) {
+        console.log("Known Failures file does not exist!");
+        return [];
+    }
+
+    // Read the file and split lines into an array
+    const data = fs.readFileSync(failedFilePath, "utf8");
+    const failures = data.split("\n").filter((line) => line.trim() !== "");
+
+    console.log("Known failures list has ", failures.length, "cases");
+
+    return failures;
+}
+
+function addFailedTestCase(
+    filePath: string,
+    fileName: string,
+    failedCase: string
+): void {
+    const failedFilePath = path.join(filePath, fileName);
+
+    // Append the failed test case to the file with a newline
+    fs.appendFileSync(failedFilePath, failedCase + "\n", "utf8");
+}