200 testing formatter idempotence

.github/workflows/main.yml

@@ -4,26 +4,41 @@
 name: Node.js CI
 
 on:
-  push:
-    branches: [ "develop" ]
-  pull_request:
-    branches: [ "develop" ]
+    push:
+        branches: ["develop"]
+    pull_request:
+        branches: ["develop"]
 
 jobs:
-  build:
-    strategy:
-      matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Install Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: 18.x
-    - run: npm install
-    - run: xvfb-run -a npm test
-      if: runner.os == 'Linux'
-    - run: npm test
-      if: runner.os != 'Linux'
+    build:
+        strategy:
+            matrix:
+                os: [windows-latest]
+        runs-on: ${{ matrix.os }}
+
+        defaults:
+            run:
+                working-directory: ${{ github.workspace }}\resources\stabilityTests
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Install Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 18.x
+            - run: npm install
+            - run: xvfb-run -a npm test
+              if: runner.os == 'Linux'
+            - run: npm test
+              if: runner.os != 'Linux'
+
+            - run: xvfb-run -a npm run get-ade-test
+              if: runner.os == 'Linux'
+            - run: npm run get-ade-test
+              if: runner.os != 'Linux'
+
+            - run: xvfb-run -a npm run stability-test
+              if: runner.os == 'Linux'
+            - run: npm run stability-test
+              if: runner.os != 'Linux'

.gitignore

@@ -7,3 +7,4 @@ node_modules
 *.vsix
 resources/ade
 resources/testResults
+resources/testResultsIdempotence

.vscode/launch.json

@@ -43,7 +43,8 @@
 				"${workspaceFolder}/out/test/**/*.js"
 			],
 			"preLaunchTask": "${defaultBuildTask}"
-		},		{
+		},
+		{
 			"name": "Stability Tests",
 			"type": "extensionHost",
 			"request": "launch",
@@ -58,10 +59,9 @@
 			"preLaunchTask": "${defaultBuildTask}"
 		}
 	],
-
 	"folders": [
 		{
 			"path": "${workspaceFolder}/resources/samples"
 		}
 	]
-}
+}

.vscode/settings.json

@@ -9,4 +9,6 @@
      "[typescript]" : {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
      },
+
+     "editor.formatOnSave": true
 }

package.json

@@ -289,7 +289,8 @@
     "copy-test-cases": "copyfiles -u 2 ./resources/samples/* .test_dir/samples",
     "copy-test-settings": "copyfiles -u 2 ./resources/samples/.vscode/* .test_dir/samples/",
     "test": "node ./out/test/runTest.js",
-    "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade"
+    "get-ade-test": "git clone https://github.com/progress/ADE.git resources/ade",
+    "stability-test": "node ./out/stability-test/runTest.js"
   },
   "devDependencies": {
     "@electron/rebuild": "^3.3.1",

src/stability-test/suite/extension.test.ts

@@ -18,6 +18,10 @@ let parserHelper: AblParserHelper;
 
 const extensionDevelopmentPath = path.resolve(__dirname, "../../../");
 const testResultsDir = join(extensionDevelopmentPath, "resources/testResults");
+const testResultsDirIdempotence = join(
+    extensionDevelopmentPath,
+    "resources/testResultsIdempotence"
+);
 
 const stabilityTestDir = join(extensionDevelopmentPath, "resources/ade");
 const extensionsToFind = [".p", ".w", ".cls", ".i"];
@@ -33,6 +37,7 @@ const testRunTimestamp = new Date()
     .replace(/[:.T-]/g, "_")
     .substring(0, 19);
 const testRunDir = join(testResultsDir, testRunTimestamp);
+const testRunDirIdempotence = join(testResultsDirIdempotence, testRunTimestamp);
 
 suite("Extension Test Suite", () => {
     console.log("Parser initialized", stabilityTestCases);
@@ -47,6 +52,7 @@ suite("Extension Test Suite", () => {
         });
 
         fs.mkdirSync(testRunDir, { recursive: true });
+        fs.mkdirSync(testRunDirIdempotence, { recursive: true });
 
         parserHelper = new AblParserHelper(
             extensionDevelopmentPath,
@@ -61,11 +67,14 @@ suite("Extension Test Suite", () => {
         );
     });
 
-    let fileId = 0;
-
     stabilityTestCases.forEach((cases) => {
-        test(`Symbol test: ${cases}`, () => {
-            stabilityTest(cases);
+        // test(`Empty space test: ${cases}`, () => {
+        //     stabilityTest(cases);
+        // }).timeout(10000);
+
+        // try to run 3 times
+        test(`Idempotence test: ${cases}`, () => {
+            runFormattingNTimes(cases, 2);
         }).timeout(10000);
     });
 });
@@ -75,8 +84,10 @@ function stabilityTest(name: string): void {
     enableFormatterDecorators();
 
     const beforeText = getInput(name);
-    const beforeCount = countActualSymbols(beforeText);
     const afterText = format(beforeText, name);
+
+    const beforeCount = countActualSymbols(beforeText);
+
     const afterCount = countActualSymbols(afterText);
 
     if (beforeCount !== afterCount) {
@@ -98,7 +109,74 @@ function stabilityTest(name: string): void {
         After: ${afterFilePath}
         `);
     }
-    // assert.strictEqual(beforeCount, afterCount);
+}
+
+function idempotenceTest(name: string) {
+    ConfigurationManager2.getInstance();
+    enableFormatterDecorators();
+
+    const beforeText = getInput(name);
+
+    const afterText = format(beforeText, name);
+
+    ConfigurationManager2.getInstance();
+    enableFormatterDecorators();
+
+    const afterText2 = format(afterText, name);
+
+    if (afterText !== afterText2) {
+        const fileName = path.basename(name, path.extname(name));
+        const afterFilePath = join(
+            testRunDirIdempotence,
+            `${fileName}_before${path.extname(name)}`
+        );
+        const after2FilePath = join(
+            testRunDirIdempotence,
+            `${fileName}_after${path.extname(name)}`
+        );
+
+        fs.writeFileSync(afterFilePath, afterText);
+        fs.writeFileSync(after2FilePath, afterText2);
+
+        assert.fail(`Text mismach
+        After: ${afterFilePath}
+        After2: ${after2FilePath}
+        `);
+    }
+}
+
+function runFormattingNTimes(name: string, n: number) {
+    let currentText = getInput(name);
+    const symbolCount = countActualSymbols(currentText);
+    for (let i = 1; i < n; i++) {
+        if (symbolCount < 500 || symbolCount > 1500) {
+            console.log("Skipping test", name);
+            return;
+        }
+        const afterText = format(currentText, name);
+        fs.writeFileSync(name, afterText);
+        if (i == n - 1 && currentText !== afterText) {
+            const fileName = path.basename(name, path.extname(name));
+            const afterFilePath = join(
+                testRunDirIdempotence,
+                `${fileName}_before${path.extname(name)}`
+            );
+            const after2FilePath = join(
+                testRunDirIdempotence,
+                `${fileName}_after${path.extname(name)}`
+            );
+
+            fs.writeFileSync(afterFilePath, currentText);
+            fs.writeFileSync(after2FilePath, afterText);
+
+            assert.fail(`Text mismach
+        After: ${afterFilePath}
+        After2: ${after2FilePath}
+        `);
+        }
+
+        currentText = afterText;
+    }
 }
 
 function getInput(fileName: string): string {