Merge pull request #294 from zong-zhe/fix-mod-missing

fix: fix missing mod when dep from local path

pkg/client/client.go

@@ -647,18 +647,12 @@ func (c *KpmClient) AddDepToPkg(kclPkg *pkg.KclPkg, d *pkg.Dependency) error {
 	}
 
 	// download all the dependencies.
-	changedDeps, _, err := c.InitGraphAndDownloadDeps(kclPkg)
+	_, _, err := c.InitGraphAndDownloadDeps(kclPkg)
 
 	if err != nil {
 		return err
 	}
 
-	// Update kcl.mod and kcl.mod.lock
-	for k, v := range changedDeps.Deps {
-		kclPkg.ModFile.Dependencies.Deps[k] = v
-		kclPkg.Dependencies.Deps[k] = v
-	}
-
 	return err
 }
 
@@ -1251,7 +1245,7 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc
 		return nil, nil, err
 	}
 
-	changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root)
+	changedDeps, err := c.downloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1283,7 +1277,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen
 }
 
 // downloadDeps will download all the dependencies of the current kcl package.
-func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[string, string], pkghome, parent string) (*pkg.Dependencies, error) {
+func (c *KpmClient) downloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependencies, depGraph graph.Graph[string, string], pkghome, parent string) (*pkg.Dependencies, error) {
 
 	newDeps := pkg.Dependencies{
 		Deps: make(map[string]pkg.Dependency),
@@ -1295,7 +1289,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			return nil, errors.InvalidDependency
 		}
 
-		existDep := c.dependencyExists(&d, &lockDeps)
+		existDep := c.dependencyExists(&d, lockDeps)
 		if existDep != nil {
 			newDeps.Deps[d.Name] = *existDep
 			continue
@@ -1328,9 +1322,10 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			}
 		}
 
-		// Update kcl.mod and kcl.mod.lock
 		newDeps.Deps[d.Name] = *lockedDep
-		lockDeps.Deps[d.Name] = *lockedDep
+		// After downloading the dependency in kcl.mod, update the dep into to the kcl.mod
+		// Only the direct dependencies are updated to kcl.mod.
+		deps.Deps[d.Name] = *lockedDep
 	}
 
 	// necessary to make a copy as when we are updating kcl.mod in below for loop
@@ -1378,20 +1373,25 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			return nil, err
 		}
 
-		// Download the dependencies.
-		nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source)
+		// Download the indirect dependencies.
+		nested, err := c.downloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source)
 		if err != nil {
 			return nil, err
 		}
 
-		// Update kcl.mod.
 		for _, d := range nested.Deps {
 			if _, ok := newDeps.Deps[d.Name]; !ok {
 				newDeps.Deps[d.Name] = d
 			}
 		}
 	}
 
+	// After each dependency is downloaded, update all the new deps to kcl.mod.lock.
+	// No matter whether the dependency is directly or indirectly.
+	for k, v := range newDeps.Deps {
+		lockDeps.Deps[k] = v
+	}
+
 	return &newDeps, nil
 }
 

pkg/client/client_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/dominikbraun/graph"
 	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/git"
@@ -1380,3 +1381,44 @@ func TestRunOciWithSettingsFile(t *testing.T) {
 	_, err = kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
 	assert.Equal(t, err, nil)
 }
+
+func TestRunGitWithLocalDep(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testPath := getTestDir("test_run_git_with_local_dep")
+	defer func() {
+		_ = os.RemoveAll(filepath.Join(testPath, "catalog"))
+	}()
+
+	testCases := []struct {
+		ref        string
+		expectFile string
+	}{
+		{"8308200", "expect1.yaml"},
+		{"0b3f5ab", "expect2.yaml"},
+	}
+
+	for _, tc := range testCases {
+
+		expectPath := filepath.Join(testPath, tc.expectFile)
+		opts := opt.DefaultCompileOptions()
+		gitOpts := git.NewCloneOptions("https://github.com/kcl-lang/flask-demo-kcl-manifests.git", tc.ref, "", "", "", nil)
+
+		result, err := kpmcli.CompileGitPkg(gitOpts, opts)
+		assert.Equal(t, err, nil)
+
+		fileBytes, err := os.ReadFile(expectPath)
+		assert.Equal(t, err, nil)
+
+		var expectObj map[string]interface{}
+		err = yaml.Unmarshal(fileBytes, &expectObj)
+		assert.Equal(t, err, nil)
+
+		var gotObj map[string]interface{}
+		err = yaml.Unmarshal([]byte(result.GetRawJsonResult()), &gotObj)
+		assert.Equal(t, err, nil)
+
+		assert.Equal(t, gotObj, expectObj)
+	}
+}

pkg/client/test_data/test_run_git_with_local_dep/expect1.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flask-demo
+  template:
+    metadata:
+      labels:
+        app: flask-demo
+    spec:
+      containers:
+      - name: flaskdemo
+        image: kcllang/flask_demo:8d31498e765ff67a2fa9933d4adffe067544b2fe
+        ports:
+        - protocol: TCP
+          containerPort: 5000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app: flask-demo
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000

pkg/client/test_data/test_run_git_with_local_dep/expect2.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flask-demo
+  template:
+    metadata:
+      labels:
+        app: flask-demo
+    spec:
+      containers:
+      - name: flaskdemo
+        image: kcllang/flask_demo:8d31498e765ff67a2fa9933d4adffe067544b2fe
+        ports:
+        - protocol: TCP
+          containerPort: 5000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app: flask-demo
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000

pkg/package/modfile.go

@@ -202,7 +202,7 @@ func (d Dependency) WithTheSameVersion(other Dependency) bool {
 
 // GetLocalFullPath will get the local path of a dependency.
 func (dep *Dependency) GetLocalFullPath(rootpath string) string {
-	if len(dep.LocalFullPath) == 0 && dep.IsFromLocal() {
+	if !filepath.IsAbs(dep.LocalFullPath) && dep.IsFromLocal() {
 		if filepath.IsAbs(dep.Source.Local.Path) {
 			return dep.Source.Local.Path
 		}

test/e2e/kpm_test.go

@@ -13,6 +13,7 @@ import (
 	"kcl-lang.io/kpm/pkg/client"
 	"kcl-lang.io/kpm/pkg/constants"
 	"kcl-lang.io/kpm/pkg/opt"
+	"kcl-lang.io/kpm/pkg/utils"
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
@@ -97,13 +98,13 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 		testSuitesRoot := filepath.Join(filepath.Join(filepath.Join(GetWorkDir(), TEST_SUITES_DIR), "kpm"), "workflows")
 
 		files, _ := os.ReadDir(testSuitesRoot)
-		for _, file := range files {
+		for _, f := range files {
+			file := f
 			ginkgo.It(filepath.Join(testSuitesRoot, file.Name()), func() {
 				if file.IsDir() {
 					testSuites := LoadAllTestSuites(filepath.Join(testSuitesRoot, file.Name()))
 					for _, ts := range testSuites {
 						ts := ts
-
 						workspace := GetWorkspace()
 
 						stdout, stderr, err := ExecKpmWithWorkDir(ts.Input, workspace)
@@ -133,10 +134,15 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 			ginkgo.It(ts.GetTestSuiteInfo(), func() {
 				workspace := GetWorkspace()
 
-				CopyDir(filepath.Join(testDataRoot, ts.Name), filepath.Join(workspace, ts.Name))
+				testSuitePath := filepath.Join(testDataRoot, ts.Name)
+				testWorkspace := workspace
+				if exist, _ := utils.Exists(testSuitePath); exist {
+					CopyDir(testSuitePath, filepath.Join(workspace, ts.Name))
+					testWorkspace = filepath.Join(workspace, ts.Name)
+				}
 
-				input := ReplaceAllKeyByValue(ts.Input, "<workspace>", filepath.Join(workspace, ts.Name))
-				stdout, stderr, err := ExecKpmWithWorkDir(input, filepath.Join(workspace, ts.Name))
+				input := ReplaceAllKeyByValue(ts.Input, "<workspace>", testWorkspace)
+				stdout, stderr, err := ExecKpmWithWorkDir(input, testWorkspace)
 
 				expectedStdout := ReplaceAllKeyByValue(ts.ExpectStdout, "<workspace>", workspace)
 				expectedStderr := ReplaceAllKeyByValue(ts.ExpectStderr, "<workspace>", workspace)

test/e2e/test_suites/kpm/kpm_add/test_kpm_add_git_commit_0/test_suite.input

@@ -0,0 +1 @@
+kpm add -git https://github.com/kcl-lang/flask-demo-kcl-manifests.git -commit 0b3f5ab

test/e2e/test_suites/kpm/kpm_add/test_kpm_add_git_commit_0/test_suite.stdout

@@ -0,0 +1,3 @@
+adding dependency 'flask-demo-kcl-manifests'
+cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests.git' with commit '0b3f5ab'
+add dependency 'flask-demo-kcl-manifests:0b3f5ab' successfully

test/e2e/test_suites/kpm/kpm_add/test_kpm_add_git_commit_1/test_suite.input

@@ -0,0 +1 @@
+kpm add -git https://github.com/kcl-lang/flask-demo-kcl-manifests.git -commit 8308200

test/e2e/test_suites/kpm/kpm_add/test_kpm_add_git_commit_1/test_suite.stdout

@@ -0,0 +1,3 @@
+adding dependency 'flask-demo-kcl-manifests'
+cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests.git' with commit '8308200'
+add dependency 'flask-demo-kcl-manifests:8308200' successfully

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_1/test_suite.input

@@ -0,0 +1 @@
+kpm run <workspace>/pkg1

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_1/test_suite.stdout

@@ -0,0 +1 @@
+The_first_kcl_program: Hello World!

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_2/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_2/test_suite.input

@@ -0,0 +1 @@
+kpm run <workspace>/pkg1

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_2/test_suite.stdout

@@ -0,0 +1,2 @@
+adding 'pkg2'
+The_first_kcl_program: Hello World!

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_3/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_3/test_suite.input

@@ -0,0 +1 @@
+kpm run <workspace>/pkg1

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_3/test_suite.stdout

@@ -0,0 +1,2 @@
+adding 'pkg2'
+The_first_kcl_program: Hello World!

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_4/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_4/test_suite.input

@@ -0,0 +1 @@
+kpm run <workspace>/pkg1

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_4/test_suite.stdout

@@ -0,0 +1,2 @@
+adding 'pkg2'
+The_first_kcl_program: Hello World!

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_5/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_5/test_suite.input

@@ -0,0 +1 @@
+kpm run <workspace>/pkg1

test/e2e/test_suites/kpm/kpm_run/test_kpm_run_multi_local_path_5/test_suite.stdout

@@ -0,0 +1,2 @@
+adding 'pkg2'
+The_first_kcl_program: Hello World!

test/e2e/test_suites/kpm/workflows/add_multi_times/1.init_an_empty_pkg/test_suite.stderr

@@ -1,3 +0,0 @@
-creating new : <workspace>/kcl.mod
-creating new : <workspace>/kcl.mod.lock
-package ' test_kpm_workspace ' init finished

test/e2e/test_suites/kpm/workflows/add_multi_times/1.init_an_empty_pkg/test_suite.stdout

@@ -0,0 +1,4 @@
+creating new :<workspace>/kcl.mod
+creating new :<workspace>/kcl.mod.lock
+creating new :<workspace>/main.k
+package 'test_kpm_workspace' init finished

test/e2e/test_suites/kpm/workflows/add_multi_times/2.first_add_konfig_dep/test_suite.stderr

@@ -1,2 +0,0 @@
-adding dependency konfig
-add dependency successfully

test/e2e/test_suites/kpm/workflows/add_multi_times/2.first_add_konfig_dep/test_suite.stdout

@@ -1 +1,3 @@
-<ignore>
+adding dependency 'konfig'
+cloning 'https://github.com/awesome-kusion/konfig.git' with tag 'v0.0.1'
+add dependency 'konfig:v0.0.1' successfully

test/e2e/test_suites/kpm/workflows/add_multi_times/3.second_add_konfig_dep/test_suite.stderr

@@ -1 +0,0 @@
-add dependency successfully

test/e2e/test_suites/kpm/workflows/add_multi_times/3.second_add_konfig_dep/test_suite.stdout

@@ -0,0 +1,2 @@
+adding dependency 'konfig'
+add dependency 'konfig:v0.0.1' successfully

test/e2e/test_suites/kpm/workflows/add_multi_times/4.third_add_konfig_dep/test_suite.stderr

@@ -1 +0,0 @@
-add dependency successfully

test/e2e/test_suites/kpm/workflows/add_multi_times/4.third_add_konfig_dep/test_suite.stdout

@@ -0,0 +1,2 @@
+adding dependency 'konfig'
+add dependency 'konfig:v0.0.1' successfully