diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 9b10246..8c6e265 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -1,20 +1,20 @@
-name: Build
-on: [push]
+name: build
+on: [push, pull_request]
 jobs:
   build:
-    name: Build qbjc and qbjc playground
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16]
+        node-version: [20]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm install --legacy-peer-deps
       - run: npm run --workspaces lint
       - run: npm run --workspaces build
       - run: npm run --workspace=qbjc test
+      - run: npm run --workspace=qbjc package-test
       - run: npm run --workspace=monaco-qb test
       - run: npm run --workspace=monaco-qb build:demo
diff --git a/qbjc/package.json b/qbjc/package.json
index 105b280..39c61f4 100644
--- a/qbjc/package.json
+++ b/qbjc/package.json
@@ -20,6 +20,7 @@
     "build:nodeRuntimeBundle": "node ./dist/tools/build-runtime-bundle.js ./dist/runtime/node-runtime-bundle-bootstrap.js ./dist/runtime/node-runtime-bundle.js",
     "build:tsc": "tsc && tsc -p tsconfig.platforms.json && chmod +x ./dist/qbjc.js",
     "build": "npm run build:grammar && npm run build:tsc && npm run build:nodeRuntimeBundle",
+    "package-test": "npm run build && ./src/tests/package-test.sh",
     "lint": "prettier --check .",
     "test": "jest"
   },
diff --git a/qbjc/src/tests/package-test.sh b/qbjc/src/tests/package-test.sh
new file mode 100755
index 0000000..3647d1c
--- /dev/null
+++ b/qbjc/src/tests/package-test.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Smoke test for verifying the published package. It runs `npm pack` and
+# verifies the output can be installed and imported.
+#
+
+TEST_SCRIPT=$(cat <<'EOF'
+
+import assert from 'assert';
+import {compile} from 'qbjc';
+import {NodeExecutor} from 'qbjc/node';
+
+async function testCompileAndRun(source: string) {
+  const {code} = await compile({
+    source,
+    sourceFileName: 'test.bas',
+  });
+  assert(code);
+  await new NodeExecutor().executeModule(code);
+}
+
+(async () => {
+  await testCompileAndRun('PRINT "Hello, World!"');
+})();
+
+EOF
+)
+SOURCE_DIR="$PWD"
+TEMP_DIR="$PWD/../tmp-smoke-test"
+
+
+cd "$SOURCE_DIR"
+echo "> Building package"
+npm pack || exit 1
+echo
+
+package_files=(*.tgz)
+if [ ${#package_files[@]} -eq 1 ]; then
+  package_file="$SOURCE_DIR/${package_files[0]}"
+  echo "> Found package $package_file"
+	echo
+else
+	echo "Could not identify package file"
+	exit 1
+fi
+
+echo "> Installing package in temp directory $TEMP_DIR"
+if [ -d "$TEMP_DIR" ]; then
+  rm -rf "$TEMP_DIR"
+fi
+mkdir -p "$TEMP_DIR"
+cd "$TEMP_DIR"
+npm init -y
+npm install --save ts-node typescript '@types/node' "$package_file"
+echo '{ "compilerOptions": { "module": "CommonJS", "esModuleInterop": true } }' > tsconfig.json
+echo
+
+echo "> Running test script"
+echo "$TEST_SCRIPT"
+if ./node_modules/.bin/ts-node -e "$TEST_SCRIPT"; then
+  echo
+	echo "> Success!"
+	exit_code=0
+else
+  exit_code=$?
+  echo
+	echo "> Error - script returned status ${exit_code}"
+fi
+echo
+
+echo "> Cleaning up"
+cd "$SOURCE_DIR"
+rm -rf "$TEMP_DIR" "$package_file"
+
+exit $exit_code