test: Execute all tests on VSIX Installation base

.github/workflows/build.yml

@@ -50,6 +50,22 @@ jobs:
         run: npm test
         if: runner.os != 'Linux'
 
+      - name: Pack for VSIX Tests
+        run: npm run package:test
+        env:
+          TEST_TEMP: ${{ runner.temp }}        
+
+      - name: Run VSIX Tests (Linux)
+        run: xvfb-run -a npm run test:vsix
+        env:
+          TEST_TEMP: ${{ runner.temp }}
+        if: runner.os == 'Linux'
+      - name: Run VSIX Tests (Win/MacOS)
+        run: npm run test:vsix
+        env:
+          TEST_TEMP: ${{ runner.temp }}        
+        if: runner.os != 'Linux'
+
       - name: Run Linter
         if: always()
         run: npm run lint

.gitignore

@@ -134,3 +134,4 @@ dist
 out
 *.vsix
 vscode*.d.ts
+tmp

.vscode-test.mjs

@@ -3,48 +3,54 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 
+
 const dirname = fileURLToPath(new URL('.', import.meta.url));
 const integrationTestDir = path.join(dirname, 'out/test/integration');
 const workspaceBaseDir = path.join(dirname, 'test-workspaces');
 
 const vsCodeVersion = process.env.VSCODE_TEST_VERSION ?? 'stable';
 const vsCodePlatform = process.env.VSCODE_TEST_PLATFORM ?? 'desktop';
 
-let createCommonOptions = (label) => {
-  if (process.env.GITHUB_ACTIONS) {
-    return {
-      platform: vsCodePlatform,
-      version: vsCodeVersion,
-      env: {
-        MOCHA_COLORS: 'true',
-        MOCHA_VSCODE_TEST: 'true'
-      },
-      mocha: {
-        ui: 'bdd',
+let extensionDevelopmentPath = '';
 
-        reporter: path.join(dirname, '.vscode-ci-test-reporter.js'),
-        reporterOption: {
-          jsonReporterOption: {
-            output: path.join(dirname, 'test-results', `${label}.json`),
-          },
-        },
-        timeout: 60_000,
-      },
-    };
-  } else {
-    return {
-      platform: vsCodePlatform,
-      version: vsCodeVersion,
-      env: {
-        MOCHA_VSCODE_TEST: 'true'
-      },
-      mocha: {
-        ui: 'bdd',
-        timeout: 60_000,
+const testMode = process.env.TEST_MODE ?? 'normal';
+
+if (testMode === 'vsix') {
+  const tempDir = process.env.TEST_TEMP ?? path.join(dirname, 'tmp')
+  extensionDevelopmentPath = path.resolve(path.join(tempDir, 'vsix', 'extension'));
+}
+
+function createCommonOptions(label) {
+  /**@type {import('@vscode/test-cli').TestConfiguration} */
+  const options = {
+    platform: vsCodePlatform,
+    version: vsCodeVersion,
+    env: {
+      MOCHA_VSCODE_TEST: 'true',
+    },
+    mocha: {
+      ui: 'bdd',
+      timeout: 60_000,
+    },
+  };
+
+  if (process.env.GITHUB_ACTIONS) {
+    options.mocha.reporter = path.join(dirname, '.vscode-ci-test-reporter.js');
+    options.mocha.reporterOption = {
+      jsonReporterOption: {
+        output: path.join(dirname, 'test-results', `${testMode}-${label}.json`),
       },
     };
+    options.env.MOCHA_COLORS = 'true';
+  }
+
+  if (extensionDevelopmentPath) {
+    options.extensionDevelopmentPath = extensionDevelopmentPath;
   }
-};
+
+
+  return options;
+}
 
 const config = [
   {

package.json

@@ -116,6 +116,7 @@
     "publish:vsce:prerelease": "npx @vscode/vsce publish --pre-release",
     "publish:ovsx:release": "npx ovsx publish",
     "publish:ovsx:prerelease": "npx ovsx publish --pre-release",
+    "package:test": "npm run compile && tsx ./scripts/prerelease.mts && npx @vscode/vsce package && tsx ./scripts/extractVsix.mts",
     "package:release": "npm run compile && tsx ./scripts/prerelease.mts && npx @vscode/vsce package",
     "package:prerelease": "npm run compile && tsx ./scripts/prerelease.mts --pre-release && npx @vscode/vsce package --pre-release",
     "vscode:prepublish": "npm run compile",
@@ -126,6 +127,7 @@
     "watch:esbuild": "npm run clean && node .esbuild.js --watch",
     "watch": "npm run clean && tsc --watch",
     "test": "tsc && vscode-test",
+    "test:vsix": "tsc && cross-env TEST_MODE=vsix vscode-test",
     "prettier": "prettier --write src",
     "lint": "eslint .",
     "fix": "eslint . --fix"
@@ -136,6 +138,7 @@
     "@types/chai": "^4.3.17",
     "@types/eslint__js": "^8.42.3",
     "@types/estree": "^1.0.5",
+    "@types/extract-zip": "^2.0.1",
     "@types/glob": "^8.1.0",
     "@types/mocha": "^10.0.8",
     "@types/node": "22.x",
@@ -155,6 +158,7 @@
     "acorn-loose": "^8.4.0",
     "ansi-colors": "^4.1.3",
     "chai": "^4.4.1",
+    "cross-env": "^7.0.3",
     "data-uri-to-buffer": "^6.0.2",
     "enhanced-resolve": "^5.17.1",
     "error-stack-parser": "^2.1.4",
@@ -163,6 +167,7 @@
     "eslint-plugin-license-header": "^0.6.1",
     "eslint-plugin-prettier": "^5.2.1",
     "eslint-visitor-keys": "^4.1.0",
+    "extract-zip": "^2.0.1",
     "glob": "^11.0.0",
     "minimatch": "^10.0.1",
     "mocha": "^10.7.3",
@@ -185,7 +190,7 @@
     "esbuild": "^0.24.0"
   },
   "mocha-vscode": {
-    "version": "1.1.0-preview+FFFFFFF",
-    "date": "2024-04-02T14:30:00Z"
+    "version": "v1.2.3+d27e65f",
+    "date": "2024-10-20T15:38:24.261Z"
   }
 }

scripts/extractVsix.mts

@@ -0,0 +1,34 @@
+/**
+ * Copyright (C) Daniel Kuschny (Danielku15) and contributors.
+ *
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+
+import extract from 'extract-zip';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const dirname = fileURLToPath(new URL('.', import.meta.url));
+
+const vsix = fs
+  .readdirSync(path.join(dirname, '..'), {
+    withFileTypes: true,
+  })
+  .filter((f) => f.name.startsWith('mocha-vscode') && f.name.endsWith('.vsix'));
+
+if (vsix.length > 1) {
+  console.error('Multiple VSIX files found, cannot decide which is the one for testing');
+  process.exit(1);
+}
+
+const tempDir = process.env.TEST_TEMP ?? path.join(dirname, '..', 'tmp')
+const extensionDir = path.join(tempDir, 'vsix');
+await fs.promises.rm(extensionDir, { recursive: true, force: true });
+await fs.promises.mkdir(extensionDir, { recursive: true });
+
+await extract(path.join(vsix[0].parentPath, vsix[0].name), {
+  dir: extensionDir,
+});