[v16] Switch JS package manager to pnpm (#44450)

* Switch JS package manager to pnpm (#44167)

* Adjust `package.json`s to pnpm

* Convert resolutions (pnpm doesn't support `**/` syntax)

It also removes resolutions that are no longer relevant.

* Convert `yarn.lock` to `pnpm-lock.yaml` with `pnpm import`

* Always add `e/web/teleport` workspace to the lockfile

* Adjust `jest` to new node_modules structure

* Adjust `storybook` to new node_modules structure

* Replace `toHaveStyleRule` with `toHaveStyle`

`jest-styled-components` package is no longer global, so `toHaveStyleRule` matchers is not recognized by TS.
We use them only in three places, so it's simpler to convert them to `toHaveStyle`.

* Adjust README files

* Adjust GHA workflows

* Adjust build assets

* Remove remaining `yarn` mentions

I don't think anyone uses the debug configuration for electron (and we don't have `start-electron` script for a long time).

* `corepack enable` -> `corepack enable pnpm`

* Automatically enable pnpm when running `make build/teleport`

* Run `pnpm import` again

* Run `pnpm dedupe`

* Add `yarn.lock` to .gitignore

* Link to pnpm installation docs

* Add a better comment for `transformIgnorePatterns`

* Make pattern for `shared` stories more specific

* Change `corepack enable pnpm` in readmes

* Run `pnpm import && pnpm dedupe` again

(cherry picked from commit 620c0a0)

* Run `pnpm import && pnpm dedupe`

* Remove `yarn.lock`

* make ensure-js-package-manager: Do not prompt before installing pnpm

---------

Co-authored-by: Rafał Cieślak <rafal.cieslak@goteleport.com>

.github/workflows/build-macos.yaml

@@ -54,9 +54,9 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
-      - name: Setup yarn
+      - name: Setup pnpm
         run: |
-          corepack enable yarn
+          corepack enable pnpm
 
       - name: Install Go Toolchain
         uses: actions/setup-go@v5

.github/workflows/lint-ui-bypass.yaml

@@ -18,7 +18,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -29,7 +29,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'

.github/workflows/lint-ui.yaml

@@ -8,7 +8,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -19,7 +19,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -39,17 +39,17 @@ jobs:
         run: |
           node --version
 
-      - name: Install Yarn dependencies
-        run: bash web/packages/build/scripts/yarn-install-frozen-lockfile.sh
+      - name: Install JS dependencies
+        run: pnpm install --frozen-lockfile
 
       - name: Build WASM
-        run: yarn build-wasm
+        run: pnpm build-wasm
 
       - name: Run Type Check
-        run: yarn type-check
+        run: pnpm type-check
 
       - name: Run lint
-        run: yarn lint
+        run: pnpm lint
 
       - name: Run Storybook smoke test
-        run: yarn storybook-smoke-test
+        run: pnpm storybook-smoke-test

.github/workflows/unit-tests-ui-bypass.yaml

@@ -19,7 +19,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -32,7 +32,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'

.github/workflows/unit-tests-ui.yaml

@@ -9,7 +9,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -22,7 +22,7 @@ on:
       - 'gen/proto/js/**'
       - 'gen/proto/ts/**'
       - 'package.json'
-      - 'yarn.lock'
+      - 'pnpm-lock.yaml'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'tsconfig.json'
@@ -46,11 +46,11 @@ jobs:
         run: |
           node --version
 
-      - name: Install Yarn dependencies
-        run: bash web/packages/build/scripts/yarn-install-frozen-lockfile.sh
+      - name: Install JS dependencies
+        run: pnpm install --frozen-lockfile
 
       - name: Build WASM
-        run: yarn build-wasm
+        run: pnpm build-wasm
 
       - name: Run tests
-        run: yarn test
+        run: pnpm test

.gitignore

@@ -103,3 +103,6 @@ web/certs/
 
 # Files produced by tests
 *.avi
+
+# We have switched to pnpm, ignore Yarn lockfile
+yarn.lock

.pnpmfile.cjs

@@ -0,0 +1,19 @@
+function afterAllResolved(lockfile, context) {
+  // The pnpm lockfile contains entries for all workspaces.
+  // This means that if the `e` repo is not cloned, we will get a different
+  // lockfile.
+  // The ugly workaround is to add an entry for e/web/teleport workspace manually.
+  // We pass an empty object because `e/web/teleport` package.json doesn't have
+  // any dependencies.
+  if (!lockfile.importers['e/web/teleport']) {
+    context.log(`Workspace 'e/web/teleport' is not present, patching lockfile.`);
+    lockfile.importers['e/web/teleport'] = {};
+  }
+  return lockfile;
+}
+
+module.exports = {
+  hooks: {
+    afterAllResolved,
+  },
+};

BUILD_macos.md

@@ -78,8 +78,8 @@ PRs with corrections and updates are welcome!
   ```
 
 * To install tools for building the UI:
-  * `brew install node yarn`
-  * Currently, [`yarn`](https://classic.yarnpkg.com/en/docs/install) (< 2.0.0) is required
+  * `brew install node corepack`
+  * `corepack enable pnpm`
   * The `Rust` and `Cargo` version in [build.assets/Makefile](https://github.com/gravitational/teleport/blob/master/build.assets/versions.mk#L11) (search for `RUST_VERSION`) are required.
   * The [`wasm-pack`](https://github.com/rustwasm/wasm-pack) version in [build.assets/Makefile](https://github.com/gravitational/teleport/blob/master/build.assets/versions.mk#L12) (search for `WASM_PACK_VERSION`) is required:
     `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`

Makefile

@@ -97,6 +97,11 @@ TARBINS = $(addprefix teleport/,$(BINS))
 CHECK_CARGO := $(shell cargo --version 2>/dev/null)
 CHECK_RUST := $(shell rustc --version 2>/dev/null)
 
+# Check if pnpm is enabled before using it
+CHECK_PNPM := $(shell pnpm --version 2>/dev/null)
+# Check if Corepack is installed before using it
+CHECK_COREPACK := $(shell corepack --version 2>/dev/null)
+
 RUST_TARGET_ARCH ?= $(CARGO_TARGET_$(OS)_$(ARCH))
 
 CARGO_TARGET_darwin_amd64 := x86_64-apple-darwin
@@ -352,7 +357,7 @@ ifeq ("$(GITHUB_REPOSITORY_OWNER)","gravitational")
 # This is done here to prevent any changes to the (BUI)LDFLAGS passed to the other binaries
 TELEPORT_LDFLAGS ?= -ldflags '$(GO_LDFLAGS) -X github.com/gravitational/teleport/lib/modules.teleportBuildType=community'
 endif
-$(BUILDDIR)/teleport: ensure-webassets bpf-bytecode rdpclient
+$(BUILDDIR)/teleport: ensure-js-package-manager ensure-webassets bpf-bytecode rdpclient
 	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "webassets_embed $(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(WEBASSETS_TAG) $(RDPCLIENT_TAG) $(PIV_BUILD_TAG) $(KUSTOMIZE_NO_DYNAMIC_PLUGIN)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) $(TELEPORT_LDFLAGS) ./tool/teleport
 
 # NOTE: Any changes to the `tsh` build here must be copied to `build.assets/windows/build.ps1`
@@ -680,16 +685,16 @@ release-windows: release-windows-unsigned
 # Makefile. Linux uses the `teleterm` target in build.assets/Makefile.
 #
 # Either CONNECT_TSH_BIN_PATH or CONNECT_TSH_APP_PATH environment variable
-# should be defined for the `yarn package-term` command to succeed. CI sets
+# should be defined for the `pnpm package-term` command to succeed. CI sets
 # this appropriately depending on whether a push build is running, or a
 # proper release (a proper release needs the APP_PATH as that points to
 # the complete signed package). See web/packages/teleterm/README.md for
 # details.
 .PHONY: release-connect
 release-connect: | $(RELEASE_DIR)
-	yarn install --frozen-lockfile
-	yarn build-term
-	yarn package-term -c.extraMetadata.version=$(VERSION) --$(ELECTRON_BUILDER_ARCH)
+	pnpm install --frozen-lockfile
+	pnpm build-term
+	pnpm package-term -c.extraMetadata.version=$(VERSION) --$(ELECTRON_BUILDER_ARCH)
 	# Only copy proper builds with tsh.app to $(RELEASE_DIR)
 	# Drop -universal "arch" from dmg name when copying to $(RELEASE_DIR)
 	if [ -n "$$CONNECT_TSH_APP_PATH" ]; then \
@@ -1629,6 +1634,28 @@ ensure-webassets-e:
 	@if [[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ]]; then mkdir -p webassets/teleport && mkdir -p webassets/e/teleport/app && cp web/packages/teleport/index.html webassets/e/teleport/index.html; \
 	else MAKE="$(MAKE)" "$(MAKE_DIR)/build.assets/build-webassets-if-changed.sh" Enterprise webassets/e/e-sha build-ui-e web e/web; fi
 
+# Enables the pnpm package manager if building webassets is not skipped,
+# pnpm is not already enabled and Corepack is available.
+# We check if pnpm is enabled, as the user may not have permission to
+# enable it, but it may already be available.
+# Enabling it merely installs a shim which then needs to be downloaded before first use.
+# Corepack typically prompts before downloading a package manager. We work around that
+# by issuing a bogus pnpm call with an env var that skips the prompt.
+.PHONY: ensure-js-package-manager
+ensure-js-package-manager:
+ifneq ($(WEBASSETS_SKIP_BUILD),1)
+	@if [ -z "$(CHECK_PNPM)" ]; then \
+		if [ -n "$(CHECK_COREPACK)" ]; then \
+			echo 'Info: pnpm is not enabled via Corepack. Enabling pnpm…'; \
+			corepack enable pnpm; \
+			COREPACK_ENABLE_DOWNLOAD_PROMPT=0 pnpm -v; \
+		else \
+			echo 'Error: Corepack is not installed, cannot enable pnpm. See the installation guide https://pnpm.io/installation#using-corepack'; \
+			exit 1; \
+		fi \
+	fi
+endif
+
 .PHONY: init-submodules-e
 init-submodules-e:
 	git submodule init e
@@ -1643,15 +1670,15 @@ backport:
 .PHONY: ensure-js-deps
 ensure-js-deps:
 	@if [[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ]]; then mkdir -p webassets/teleport && touch webassets/teleport/index.html; \
-	else yarn install --ignore-scripts; fi
+	else pnpm install --frozen-lockfile; fi
 
 .PHONY: build-ui
 build-ui: ensure-js-deps
-	@[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ] || yarn build-ui-oss
+	@[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ] || pnpm build-ui-oss
 
 .PHONY: build-ui-e
 build-ui-e: ensure-js-deps
-	@[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ] || yarn build-ui-e
+	@[ "${WEBASSETS_SKIP_BUILD}" -eq 1 ] || pnpm build-ui-e
 
 .PHONY: docker-ui
 docker-ui:

README.md

@@ -149,8 +149,8 @@ Ensure you have installed correct versions of necessary dependencies:
   (search for `RUST_VERSION`)
 * For `tsh` version > `10.x` with FIDO support, you will need `libfido` and `openssl 1.1` installed locally
 * To build the web UI:
-  * [`yarn`](https://classic.yarnpkg.com/en/docs/install)(< 2.0.0) is required.
-  * If you prefer not to install/use yarn, but have docker available, you can run `make docker-ui` instead.
+  * [`pnpm`](https://pnpm.io/installation#using-corepack). If you have Node.js installed, run `corepack enable pnpm` to make `pnpm` available.
+  * If you prefer not to install/use pnpm, but have docker available, you can run `make docker-ui` instead.
   * The `Rust` and `Cargo` version in [build.assets/Makefile](https://github.com/gravitational/teleport/blob/master/build.assets/versions.mk#L11) (search for `RUST_VERSION`) are required.
   * The [`wasm-pack`](https://github.com/rustwasm/wasm-pack) version in [build.assets/Makefile](https://github.com/gravitational/teleport/blob/master/build.assets/versions.mk#L12) (search for `WASM_PACK_VERSION`) is required.
   * [`binaryen`](https://github.com/WebAssembly/binaryen) (which contains `wasm-opt`) is required to be installed manually

build.assets/Dockerfile-arm

@@ -57,7 +57,7 @@ RUN NODE_ARCH="$(if [ "$BUILDARCH" = 'amd64' ]; then echo 'x64'; else echo 'arm6
     curl -o "$NODE_FILE" -fsSL "$NODE_URL" && \
     tar -xJf "$NODE_FILE" -C /usr/local/lib/nodejs-linux --strip-components=1 && \
     rm -f "$NODE_FILE"
-RUN corepack enable yarn
+RUN corepack enable pnpm
 
 # Install Go.
 ARG GOLANG_VERSION

build.assets/Dockerfile-node

@@ -20,6 +20,8 @@ FROM node:${NODE_VERSION}-buster AS buildbox
 # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
 ARG BUILDARCH
 
+RUN corepack enable pnpm
+
 COPY locale.gen /etc/locale.gen
 COPY profile /etc/profile
 ENV LANGUAGE="en_US.UTF-8" \

build.assets/Makefile

@@ -247,16 +247,16 @@ buildbox-node:
 .PHONY:teleterm
 teleterm: buildbox-node
 	docker run $(DOCKERFLAGS) $(NOROOT) $(BUILDBOX_NODE) \
-		bash -c "cd $(SRCDIR) && export CONNECT_TSH_BIN_PATH=\$$PWD/../teleport/build/tsh && yarn install --frozen-lockfile && yarn build-term && yarn package-term -c.extraMetadata.version=$(CONNECT_VERSION)"
+		bash -c "cd $(SRCDIR) && export CONNECT_TSH_BIN_PATH=\$$PWD/../teleport/build/tsh && pnpm install --frozen-lockfile && pnpm build-term && pnpm package-term -c.extraMetadata.version=$(CONNECT_VERSION)"
 
 # Builds webassets inside Docker.
 .PHONY:ui
 ui: buildbox
 	docker run -u $(UID):$(GID) -v "$$(pwd)/../":/teleport $(BUILDBOX) \
-		bash -c "cd ../teleport && yarn install --frozen-lockfile && yarn build-ui"
+		bash -c "cd ../teleport && pnpm install --frozen-lockfile && pnpm build-ui"
 
 # Builds webassets inside the buildbox-node container via the
-# Makefile (instead of calling yarn directly as the ui target above does)
+# Makefile (instead of calling pnpm directly as the ui target above does)
 # as that computes a SHA to prevent unnecessary rebuilds. This is used by
 # the CentOS 7 build that does not have a new enough version of node.js
 # to build webassets. Enterprise webassets will only be built if the e

build.assets/build-webassets-if-changed.sh

@@ -55,14 +55,14 @@ for i in "${!SRC_DIRECTORIES[@]}"; do
   SRC_DIRECTORIES[i]="$ROOT_PATH/${SRC_DIRECTORIES[i]}"
 done
 
-# Calculate the current hash-of-hashes of the given source directories, package.json, and yarn.lock.
-# We exclude node_modules as it's covered by package.json and yarn.lock.
+# Calculate the current hash-of-hashes of the given source directories, package.json, and pnpm-lock.yaml.
+# We exclude node_modules as it's covered by package.json and pnpm-lock.yaml.
 # We also exclude .swc as it's a cache directory for the swc compiler,
 # and ironrdp/pkg as it's filled with the generated wasm files.
 function calculate_sha() {
   #shellcheck disable=SC2086
   #We want to split $SHASUM on spaces so we dont want it quoted.
-  find "${SRC_DIRECTORIES[@]}" "$ROOT_PATH/package.json" "$ROOT_PATH/yarn.lock" \
+  find "${SRC_DIRECTORIES[@]}" "$ROOT_PATH/package.json" "$ROOT_PATH/pnpm-lock.yaml" \
     -not \( -type d \( -name node_modules -o -name .swc -o -path '*ironrdp/pkg*' \) -prune \) \
     -type f -print0 | \
     LC_ALL=C sort -z | xargs -0 $SHASUM | awk '{print $1}' | $SHASUM | tr -d ' -'
@@ -86,10 +86,10 @@ if [ "$BUILD" = "true" ]; then \
   print_for_user "detected changes in $TYPE webassets. Rebuilding..."
   "$MAKE" -C "$ROOT_PATH" "$BUILD_TARGET"; \
   # Recalculate the current SHA and record into the LAST_SHA_FILE. The make target is expected to have
-  # created any necessary directories here. The recalculation is necessary as yarn.lock may have been
+  # created any necessary directories here. The recalculation is necessary as pnpm-lock.yaml may have been
   # updated by the build process.
   mkdir -p "$(dirname "$LAST_SHA_FILE")"
-  # Save SHA with yarn.lock before yarn install
+  # Save SHA with pnpm-lock.yaml before installing dependencies.
   echo "$CURRENT_SHA" > "$LAST_SHA_FILE"
   print_for_user "$TYPE webassets successfully updated."
 else

build.assets/windows/build.ps1

@@ -144,7 +144,7 @@ function Install-Node {
         Expand-Archive -Path $NodeZipfile -DestinationPath $ToolchainDir
         Rename-Item -Path "$ToolchainDir/node-v$NodeVersion-win-x64" -NewName "$ToolchainDir/node"
         Enable-Node -ToolchainDir $ToolchainDir
-        corepack enable yarn
+        corepack enable pnpm
         Write-Host "::endgroup::"
     }
 }
@@ -435,9 +435,9 @@ function Build-Connect {
     $CommandDuration = Measure-Block {
         Write-Host "::group::Building Teleport Connect..."
         $env:CONNECT_TSH_BIN_PATH = "$SignedTshBinaryPath"
-        yarn install --frozen-lockfile
-        yarn build-term
-        yarn package-term "-c.extraMetadata.version=$TeleportVersion"
+        pnpm install --frozen-lockfile
+        pnpm build-term
+        pnpm package-term "-c.extraMetadata.version=$TeleportVersion"
         $BinaryName = "Teleport Connect Setup-$TeleportVersion.exe"
         Invoke-SignBinary -UnsignedBinaryPath "$TeleportSourceDirectory\web\packages\teleterm\build\release\$BinaryName" `
             -SignedBinaryPath "$ArtifactDirectory\$BinaryName"

jest.config.js

@@ -8,12 +8,14 @@ const esModules = [
   'd3-color',
   'd3-scale',
   'd3-interpolate',
+  'd3-time-format',
   'd3-array',
   'd3-format',
   'd3-time',
   'd3-shape',
   'd3-path',
   'internmap',
+  '@nivo/bar',
 ].join('|');
 
 /** @type {import('@jest/types').Config.InitialOptions} */
@@ -27,7 +29,12 @@ module.exports = {
     // '**/packages/design/src/**/*.jsx',
     '**/packages/shared/components/**/*.jsx',
   ],
-  transformIgnorePatterns: [`/node_modules/(?!${esModules})`],
+  // Copied from https://jestjs.io/docs/configuration#transformignorepatterns-arraystring
+  // Because in pnpm packages are symlinked to node_modules/.pnpm,
+  // we need to transform packages in that directory.
+  // We use a relative pattern to match the second 'node_modules/' in
+  // 'node_modules/.pnpm/@scope+pkg-b@x.x.x/node_modules/@scope/pkg-b/'.
+  transformIgnorePatterns: [`node_modules/(?!.pnpm|${esModules})`],
   coverageReporters: ['text-summary', 'lcov'],
   testPathIgnorePatterns: ['e2e'],
   setupFilesAfterEnv: [