perf(cli): parse package.json and pnpm-lock.yaml using streaming io (…

…#7296) This avoids putting the entire lockfile into memory all at once just to iterate over it once and throw it away. --- ### Changes are visible to end-users: yes - Searched for relevant documentation and updated as needed: yes - Breaking change (forces users to change their own code or config): no - Suggested release notes appear below: yes Memory improvements to js `aspect configure` ### Test plan - Covered by existing test cases GitOrigin-RevId: ac5ef37ffcdba9db4108632d62fdfc9f4de8b981
aspect-build · Nov 30, 2024 · 8616e51 · 8616e51
1 parent c98f5b7
commit 8616e51
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 54 deletions.
diff --git a/gazelle/js/node/package.go b/gazelle/js/node/package.go
@@ -19,17 +19,15 @@ type npmPackageJSON struct {
 // Extract the various import types from the package.json file such as
 // 'main' and 'exports' fields.
 func ParsePackageJsonImportsFile(rootDir, packageJsonPath string) ([]string, error) {
-	content, err := os.ReadFile(path.Join(rootDir, packageJsonPath))
+	packageJsonReader, err := os.Open(path.Join(rootDir, packageJsonPath))
 	if err != nil {
 		return nil, err
 	}
 
-	return parsePackageJsonImports(content)
-}
+	packageJsonDecoder := jsonr.NewDecoder(packageJsonReader)
 
-func parsePackageJsonImports(packageJsonContent []byte) ([]string, error) {
 	var c npmPackageJSON
-	if err := jsonr.Unmarshal(packageJsonContent, &c); err != nil {
+	if err := packageJsonDecoder.Decode(&c); err != nil {
 		return nil, err
 	}
 

diff --git a/gazelle/js/pnpm/parser.go b/gazelle/js/pnpm/parser.go
@@ -1,7 +1,9 @@
 package gazelle
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"regexp"
@@ -14,12 +16,12 @@ type WorkspacePackageVersionMap map[string]map[string]string
 /* Parse a lockfile and return a map of workspace projects to a map of dependency name to version.
  */
 func ParsePnpmLockFileDependencies(lockfilePath string) WorkspacePackageVersionMap {
-	yamlFileContent, readErr := os.ReadFile(lockfilePath)
+	yamlFileReader, readErr := os.Open(lockfilePath)
 	if readErr != nil {
 		log.Fatalf("failed to read lockfile '%s': %s", lockfilePath, readErr.Error())
 	}
 
-	deps, err := parsePnpmLockDependencies(yamlFileContent)
+	deps, err := parsePnpmLockDependencies(yamlFileReader)
 	if err != nil {
 		log.Fatalf("pnpm parse - %v\n", err)
 	}
@@ -28,23 +30,33 @@ func ParsePnpmLockFileDependencies(lockfilePath string) WorkspacePackageVersionM
 
 var lockVersionRegex = regexp.MustCompile(`^\s*lockfileVersion: '?(?P<Version>\d\.\d)'?`)
 
-func parsePnpmLockVersion(yamlFileContent []byte) (string, error) {
-	match := lockVersionRegex.FindSubmatch(yamlFileContent)
+func parsePnpmLockVersion(yamlFileReader *bufio.Reader) (string, error) {
+	versionBytes, isShort, err := yamlFileReader.ReadLine()
+
+	if isShort {
+		return "", fmt.Errorf("failed to read lockfile version, line too long: '%s...'", string(versionBytes))
+	}
+	if err == io.EOF {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("failed to read lockfile version: %v", err)
+	}
+
+	match := lockVersionRegex.FindSubmatch(versionBytes)
 
 	if len(match) != 2 {
-		return "", fmt.Errorf("failed to find lockfile version in: %q", string(yamlFileContent))
+		return "", fmt.Errorf("failed to find lockfile version in: %q", string(versionBytes))
 	}
 
 	return string(match[1]), nil
 }
 
-func parsePnpmLockDependencies(yamlFileContent []byte) (WorkspacePackageVersionMap, error) {
-	if len(yamlFileContent) == 0 {
-		return WorkspacePackageVersionMap{}, nil
-	}
+func parsePnpmLockDependencies(yamlFileReader io.Reader) (WorkspacePackageVersionMap, error) {
+	yamlReader := bufio.NewReader(yamlFileReader)
 
-	versionStr, versionErr := parsePnpmLockVersion(yamlFileContent)
-	if versionErr != nil {
+	versionStr, versionErr := parsePnpmLockVersion(yamlReader)
+	if versionStr == "" || versionErr != nil {
 		return nil, versionErr
 	}
 
@@ -54,11 +66,11 @@ func parsePnpmLockDependencies(yamlFileContent []byte) (WorkspacePackageVersionM
 	}
 
 	if version.Major() == 5 {
-		return parsePnpmLockDependenciesV5(yamlFileContent)
+		return parsePnpmLockDependenciesV5(yamlReader)
 	} else if version.Major() == 6 {
-		return parsePnpmLockDependenciesV6(yamlFileContent)
+		return parsePnpmLockDependenciesV6(yamlReader)
 	} else if version.Major() == 9 {
-		return parsePnpmLockDependenciesV9(yamlFileContent)
+		return parsePnpmLockDependenciesV9(yamlReader)
 	}
 
 	return nil, fmt.Errorf("unsupported version: %v", versionStr)

diff --git a/gazelle/js/pnpm/parser_test.go b/gazelle/js/pnpm/parser_test.go
@@ -1,19 +1,21 @@
 package gazelle
 
 import (
+	"bufio"
+	"strings"
 	"testing"
 )
 
 func TestPnpmLockParseDependencies(t *testing.T) {
 	t.Run("lockfile version", func(t *testing.T) {
-		v, e := parsePnpmLockVersion([]byte("lockfileVersion: 5.4"))
+		v, e := parsePnpmLockVersion(bufio.NewReader(strings.NewReader("lockfileVersion: 5.4")))
 		if e != nil {
 			t.Error(e)
 		} else if v != "5.4" {
 			t.Error("Failed to parse lockfile version 5.4")
 		}
 
-		v, e = parsePnpmLockVersion([]byte("lockfileVersion: '6.0'"))
+		v, e = parsePnpmLockVersion(bufio.NewReader(strings.NewReader("lockfileVersion: '6.0'")))
 		if e != nil {
 			t.Error(e)
 		} else if v != "6.0" {
@@ -22,35 +24,34 @@ func TestPnpmLockParseDependencies(t *testing.T) {
 	})
 
 	t.Run("empty lock file", func(t *testing.T) {
-		emptyLock, err := parsePnpmLockDependencies([]byte(""))
+		emptyLock, err := parsePnpmLockDependencies(strings.NewReader(""))
 		if err != nil {
 			t.Error("Parse failure: ", err)
 		}
-		if emptyLock == nil {
-			t.Error("Empty lock file not parsed")
+		if emptyLock != nil {
+			t.Errorf("Empty lock file returned non-nil, got: %v", emptyLock)
 		}
 	})
 
 	t.Run("unsupported version", func(t *testing.T) {
-		_, err := parsePnpmLockDependencies([]byte("lockfileVersion: 4.0"))
+		_, err := parsePnpmLockDependencies(strings.NewReader("lockfileVersion: 4.0"))
 		if err == nil {
 			t.Error("Expected error for unsupported version (4.0)")
 		}
 
-		_, err2 := parsePnpmLockDependencies([]byte("lockfileVersion: '4.0'"))
+		_, err2 := parsePnpmLockDependencies(strings.NewReader("lockfileVersion: '4.0'"))
 		if err2 == nil {
 			t.Error("Expected error for unsupported version ('4.0')")
 		}
 
-		_, err3 := parsePnpmLockDependencies([]byte("lockfileVersion: 10.0"))
+		_, err3 := parsePnpmLockDependencies(strings.NewReader("lockfileVersion: 10.0"))
 		if err3 == nil {
 			t.Error("Expected error for unsupported version (10)")
 		}
 	})
 
 	t.Run("basic deps (lockfile v5)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: 5.4
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: 5.4
 
 specifiers:
   '@aspect-test/a': 5.0.2
@@ -85,8 +86,7 @@ peerDependencies:
 	})
 
 	t.Run("basic deps (lockfile v6)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: '6.0'
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: '6.0'
 
 dependencies:
   '@aspect-test/a':
@@ -120,8 +120,7 @@ devDependencies:
 	})
 
 	t.Run("basic deps (lockfile v9)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: '9.0'
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: '9.0'
 
 dependencies:
   '@aspect-test/a':
@@ -155,8 +154,7 @@ devDependencies:
 	})
 
 	t.Run("basic deps in single project workspace (lockfile v6)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: '6.0'
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: '6.0'
 
 importers:
   .:
@@ -191,8 +189,7 @@ importers:
 	})
 
 	t.Run("basic deps in single project workspace (lockfile v9)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: '9.0'
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: '9.0'
 
 importers:
   .:
@@ -227,8 +224,7 @@ importers:
 	})
 
 	t.Run("basic deps in single project workspace (lockfile v5)", func(t *testing.T) {
-		basic, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: 5.4
+		basic, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: 5.4
 
 importers:
   .:
@@ -261,22 +257,20 @@ importers:
 	})
 
 	t.Run("no deps property", func(t *testing.T) {
-		empty, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: 5.4
+		empty, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: 5.4
 `))
 
 		if err != nil {
 			t.Error("Parse failure: ", err)
 		}
 
-		if len(empty) != 1 || len(empty["."]) != 0 {
+		if len(empty) != 0 {
 			t.Error("No deps parse error: ", empty)
 		}
 	})
 
 	t.Run("deps to workspace pkgs (lockfile v5)", func(t *testing.T) {
-		wksps, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: 5.3
+		wksps, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: 5.3
 importers:
   a:
     specifiers:
@@ -314,8 +308,7 @@ importers:
 	})
 
 	t.Run("deps to workspace pkgs (lockfile v6)", func(t *testing.T) {
-		wksps, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: '6.1'
+		wksps, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: '6.1'
 importers:
   a:
     dependencies:
@@ -353,8 +346,7 @@ importers:
 	})
 
 	t.Run("workspace deps (lockfile v5)", func(t *testing.T) {
-		wksps, err := parsePnpmLockDependencies([]byte(`
-lockfileVersion: 5.4
+		wksps, err := parsePnpmLockDependencies(strings.NewReader(`lockfileVersion: 5.4
 importers:
   .:
     specifiers:

diff --git a/gazelle/js/pnpm/parser_v5.go b/gazelle/js/pnpm/parser_v5.go
@@ -2,13 +2,17 @@ package gazelle
 
 import (
 	"fmt"
+	"io"
 
 	"gopkg.in/yaml.v3"
 )
 
-func parsePnpmLockDependenciesV5(yamlFileContent []byte) (WorkspacePackageVersionMap, error) {
+func parsePnpmLockDependenciesV5(yamlReader io.Reader) (WorkspacePackageVersionMap, error) {
 	lockfile := PnpmLockfileV5{}
-	unmarshalErr := yaml.Unmarshal(yamlFileContent, &lockfile)
+	unmarshalErr := yaml.NewDecoder(yamlReader).Decode(&lockfile)
+	if unmarshalErr == io.EOF {
+		return nil, nil
+	}
 	if unmarshalErr != nil {
 		return nil, fmt.Errorf("parse error: %v", unmarshalErr)
 	}

diff --git a/gazelle/js/pnpm/parser_v6.go b/gazelle/js/pnpm/parser_v6.go
@@ -2,13 +2,17 @@ package gazelle
 
 import (
 	"fmt"
+	"io"
 
 	"gopkg.in/yaml.v3"
 )
 
-func parsePnpmLockDependenciesV6(yamlFileContent []byte) (WorkspacePackageVersionMap, error) {
+func parsePnpmLockDependenciesV6(yamlReader io.Reader) (WorkspacePackageVersionMap, error) {
 	lockfile := PnpmLockfileV6{}
-	unmarshalErr := yaml.Unmarshal(yamlFileContent, &lockfile)
+	unmarshalErr := yaml.NewDecoder(yamlReader).Decode(&lockfile)
+	if unmarshalErr == io.EOF {
+		return nil, nil
+	}
 	if unmarshalErr != nil {
 		return nil, fmt.Errorf("parse error: %v", unmarshalErr)
 	}

diff --git a/gazelle/js/pnpm/parser_v9.go b/gazelle/js/pnpm/parser_v9.go
@@ -1,6 +1,8 @@
 package gazelle
 
-func parsePnpmLockDependenciesV9(yamlFileContent []byte) (WorkspacePackageVersionMap, error) {
+import "io"
+
+func parsePnpmLockDependenciesV9(yamlReader io.Reader) (WorkspacePackageVersionMap, error) {
 	// The top-level lockfile object is the same as v6 for the WorkspacePackageVersionMap requirements
-	return parsePnpmLockDependenciesV6(yamlFileContent)
+	return parsePnpmLockDependenciesV6(yamlReader)
 }