From 688928294e2c425a1b213b37d41507781d25b17f Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Fri, 10 May 2024 21:34:03 +0530 Subject: [PATCH 1/8] feat: add kpm pkg include and exclude Signed-off-by: Asish Kumar --- pkg/client/client.go | 3 ++- pkg/package/modfile.go | 10 ++++++---- pkg/package/package.go | 10 ++++++++++ pkg/package/test_data/test_data_toml/expected.toml | 2 ++ pkg/package/toml.go | 11 +++++++++++ pkg/package/toml_test.go | 2 ++ pkg/utils/utils.go | 14 +++++++++++++- pkg/utils/utils_test.go | 4 ++-- 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index c4e73c53..fa827b34 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -696,7 +696,8 @@ func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool) } // Tar the current kcl package into a "*.tar" file. - err := utils.TarDir(kclPkg.HomePath, tarPath) + fmt.Println(kclPkg.GetPkgName()) + err := utils.TarDir(kclPkg.HomePath, kclPkg.GetPkgInclude(), kclPkg.GetPkgExclude(), tarPath) if err != nil { return reporter.NewErrorEvent(reporter.FailedPackage, err, "failed to package the kcl module") } diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 95a5832d..9c59f06f 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -28,10 +28,12 @@ const ( // 'Package' is the kcl package section of 'kcl.mod'. type Package struct { - Name string `toml:"name,omitempty"` // kcl package name - Edition string `toml:"edition,omitempty"` // kcl compiler version - Version string `toml:"version,omitempty"` // kcl package version - Description string `toml:"description,omitempty"` // kcl package description + Name string `toml:"name,omitempty"` // kcl package name + Edition string `toml:"edition,omitempty"` // kcl compiler version + Version string `toml:"version,omitempty"` // kcl package version + Description string `toml:"description,omitempty"` // kcl package description + Include []string `toml:"include,omitempty"` // kcl package include + Exclude []string `toml:"exclude,omitempty"` // kcl package exclude } // 'ModFile' is kcl package file 'kcl.mod'. diff --git a/pkg/package/package.go b/pkg/package/package.go index 91dd547d..9eba0634 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -288,6 +288,16 @@ func (KclPkg *KclPkg) GetPkgDescription() string { return KclPkg.ModFile.Pkg.Description } +// GetPkgInclude returns the include of package. +func (KclPkg *KclPkg) GetPkgInclude() []string { + return KclPkg.ModFile.Pkg.Include +} + +// GetPkgExclude returns the exclude of package. +func (KclPkg *KclPkg) GetPkgExclude() []string { + return KclPkg.ModFile.Pkg.Exclude +} + // GenCheckSum generates the checksum of the current kcl package. func (KclPkg *KclPkg) GenCheckSum() (string, error) { return utils.HashDir(KclPkg.HomePath) diff --git a/pkg/package/test_data/test_data_toml/expected.toml b/pkg/package/test_data/test_data_toml/expected.toml index 4d9b4c16..7bb599e9 100644 --- a/pkg/package/test_data/test_data_toml/expected.toml +++ b/pkg/package/test_data/test_data_toml/expected.toml @@ -2,6 +2,8 @@ name = "MyKcl" edition = "v0.0.1" version = "v0.0.1" +include = ["src/", "README.md", "LICENSE"] +exclude = ["target/", ".git/", "*.log"] [dependencies] MyOciKcl1 = "0.0.1" diff --git a/pkg/package/toml.go b/pkg/package/toml.go index 6bc8a34e..14652665 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -231,6 +231,8 @@ const NAME_FLAG = "name" const EDITION_FLAG = "edition" const VERSION_FLAG = "version" const DESCRIPTION_FLAG = "description" +const INCLUDE_FLAG = "include" +const EXCLUDE_FLAG = "exclude" func (pkg *Package) UnmarshalTOML(data interface{}) error { meta, ok := data.(map[string]interface{}) @@ -253,6 +255,15 @@ func (pkg *Package) UnmarshalTOML(data interface{}) error { if v, ok := meta[DESCRIPTION_FLAG].(string); ok { pkg.Description = v } + + if v, ok := meta[INCLUDE_FLAG].([]string); ok { + pkg.Include = v + } + + if v, ok := meta[EXCLUDE_FLAG].([]string); ok { + pkg.Exclude = v + } + return nil } diff --git a/pkg/package/toml_test.go b/pkg/package/toml_test.go index 0b0e78c3..7be5956f 100644 --- a/pkg/package/toml_test.go +++ b/pkg/package/toml_test.go @@ -77,6 +77,8 @@ func TestUnMarshalTOML(t *testing.T) { assert.Equal(t, modfile.Pkg.Name, "MyKcl") assert.Equal(t, modfile.Pkg.Edition, "v0.0.1") assert.Equal(t, modfile.Pkg.Version, "v0.0.1") + assert.Equal(t, modfile.Pkg.Include, []string{"src/", "README.md", "LICENSE"}) + assert.Equal(t, modfile.Pkg.Exclude, []string{"target/", ".git/", "*.log"}) assert.Equal(t, len(modfile.Dependencies.Deps), 2) assert.NotEqual(t, modfile.Dependencies.Deps["MyKcl1"], nil) assert.Equal(t, modfile.Dependencies.Deps["MyKcl1"].Name, "MyKcl1") diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 02fd9c65..43416a08 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -126,7 +126,7 @@ func Exists(path string) (bool, error) { // todo: Consider using the OCI tarball as the standard tar format. var ignores = []string{".git", ".tar"} -func TarDir(srcDir string, tarPath string) error { +func TarDir(srcDir string, include []string, exclude []string, tarPath string) error { fw, err := os.Create(tarPath) if err != nil { @@ -148,6 +148,18 @@ func TarDir(srcDir string, tarPath string) error { } } + for _, ex := range exclude { + if strings.Contains(path, ex) { + return nil + } + } + + for _, inc := range include { + if strings.Contains(path, inc) { + return nil + } + } + relPath, _ := filepath.Rel(srcDir, path) relPath = filepath.ToSlash(relPath) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 2ab8a51b..e933f0e6 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -74,8 +74,8 @@ func TestTarDir(t *testing.T) { if !os.IsNotExist(err) { os.Remove(tarPath) } - - err = TarDir(filepath.Join(testDir, "test_src"), tarPath) + emptyArrayOfStrings := []string{} + err = TarDir(filepath.Join(testDir, "test_src"), emptyArrayOfStrings, emptyArrayOfStrings, tarPath) assert.Equal(t, err, nil) _, err = os.Stat(tarPath) From 2d1a654463f61b4c5b189618db3cc1c554494606 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Fri, 10 May 2024 21:35:22 +0530 Subject: [PATCH 2/8] removed print statements --- pkg/client/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index fa827b34..c8944301 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -696,7 +696,6 @@ func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool) } // Tar the current kcl package into a "*.tar" file. - fmt.Println(kclPkg.GetPkgName()) err := utils.TarDir(kclPkg.HomePath, kclPkg.GetPkgInclude(), kclPkg.GetPkgExclude(), tarPath) if err != nil { return reporter.NewErrorEvent(reporter.FailedPackage, err, "failed to package the kcl module") From f9a58dd3f1e564daabe10008c4eec8f71f8e0e5a Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 12:36:22 +0530 Subject: [PATCH 3/8] feat: updated toml.go Signed-off-by: Asish Kumar --- pkg/package/toml.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/package/toml.go b/pkg/package/toml.go index 14652665..904a3b22 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -256,12 +256,20 @@ func (pkg *Package) UnmarshalTOML(data interface{}) error { pkg.Description = v } - if v, ok := meta[INCLUDE_FLAG].([]string); ok { - pkg.Include = v + convertToStringArray := func(v interface{}) []string { + var arr []string + for _, item := range v.([]interface{}) { + arr = append(arr, item.(string)) + } + return arr + } + + if v, ok := meta[INCLUDE_FLAG].([]interface{}); ok { + pkg.Include = convertToStringArray(v) } - if v, ok := meta[EXCLUDE_FLAG].([]string); ok { - pkg.Exclude = v + if v, ok := meta[EXCLUDE_FLAG].([]interface{}); ok { + pkg.Exclude = convertToStringArray(v) } return nil From ee93448f0d522cf6bc2b63c2af541f0c2737ca0b Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 13:42:08 +0530 Subject: [PATCH 4/8] feat: updated utils.go Signed-off-by: Asish Kumar --- pkg/utils/utils.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 43416a08..d7c1d545 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -137,6 +137,8 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e tw := tar.NewWriter(fw) defer tw.Close() + fmt.Println(exclude) + err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -148,6 +150,8 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e } } + fmt.Println(path) + for _, ex := range exclude { if strings.Contains(path, ex) { return nil @@ -155,8 +159,8 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e } for _, inc := range include { - if strings.Contains(path, inc) { - return nil + if !strings.Contains(path, inc) { + return nil } } From 7239ec4542ef600f51d4fceb2dbb538de1374567 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 13:46:23 +0530 Subject: [PATCH 5/8] feat: removed print statement from utils.go Signed-off-by: Asish Kumar --- pkg/utils/utils.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index d7c1d545..158e6ca6 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -150,8 +150,6 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e } } - fmt.Println(path) - for _, ex := range exclude { if strings.Contains(path, ex) { return nil From 4b99ed8454621f033e6882e2f1868bbaae11605a Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 16:55:36 +0530 Subject: [PATCH 6/8] fix: addressed comments Signed-off-by: Asish Kumar --- pkg/client/client.go | 2 +- pkg/package/toml_test.go | 2 +- pkg/utils/utils.go | 17 +++++++++++------ pkg/utils/utils_test.go | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index c8944301..a2a17a56 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -696,7 +696,7 @@ func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool) } // Tar the current kcl package into a "*.tar" file. - err := utils.TarDir(kclPkg.HomePath, kclPkg.GetPkgInclude(), kclPkg.GetPkgExclude(), tarPath) + err := utils.TarDir(kclPkg.HomePath, tarPath, kclPkg.GetPkgInclude(), kclPkg.GetPkgExclude()) if err != nil { return reporter.NewErrorEvent(reporter.FailedPackage, err, "failed to package the kcl module") } diff --git a/pkg/package/toml_test.go b/pkg/package/toml_test.go index 7be5956f..71f9eca5 100644 --- a/pkg/package/toml_test.go +++ b/pkg/package/toml_test.go @@ -78,7 +78,7 @@ func TestUnMarshalTOML(t *testing.T) { assert.Equal(t, modfile.Pkg.Edition, "v0.0.1") assert.Equal(t, modfile.Pkg.Version, "v0.0.1") assert.Equal(t, modfile.Pkg.Include, []string{"src/", "README.md", "LICENSE"}) - assert.Equal(t, modfile.Pkg.Exclude, []string{"target/", ".git/", "*.log"}) + assert.Equal(t, modfile.Pkg.Exclude, []string{"target/", ".git/", "*.log"}) assert.Equal(t, len(modfile.Dependencies.Deps), 2) assert.NotEqual(t, modfile.Dependencies.Deps["MyKcl1"], nil) assert.Equal(t, modfile.Dependencies.Deps["MyKcl1"].Name, "MyKcl1") diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 158e6ca6..2c6e8151 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -126,8 +126,7 @@ func Exists(path string) (bool, error) { // todo: Consider using the OCI tarball as the standard tar format. var ignores = []string{".git", ".tar"} -func TarDir(srcDir string, include []string, exclude []string, tarPath string) error { - +func TarDir(srcDir string, tarPath string, include []string, exclude []string) error { fw, err := os.Create(tarPath) if err != nil { log.Fatal(err) @@ -137,8 +136,6 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e tw := tar.NewWriter(fw) defer tw.Close() - fmt.Println(exclude) - err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -150,14 +147,22 @@ func TarDir(srcDir string, include []string, exclude []string, tarPath string) e } } + getNewPattern := func(ex string) string { + newPath := ex + if !strings.HasPrefix(ex, srcDir + "/") { + newPath = srcDir + "/" + ex + } + return newPath + } + for _, ex := range exclude { - if strings.Contains(path, ex) { + if matched, _ := filepath.Match(getNewPattern(ex), path); matched { return nil } } for _, inc := range include { - if !strings.Contains(path, inc) { + if matched, _ := filepath.Match(getNewPattern(inc), path); !matched { return nil } } diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index e933f0e6..35d77952 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -75,7 +75,7 @@ func TestTarDir(t *testing.T) { os.Remove(tarPath) } emptyArrayOfStrings := []string{} - err = TarDir(filepath.Join(testDir, "test_src"), emptyArrayOfStrings, emptyArrayOfStrings, tarPath) + err = TarDir(filepath.Join(testDir, "test_src"), tarPath, emptyArrayOfStrings, emptyArrayOfStrings) assert.Equal(t, err, nil) _, err = os.Stat(tarPath) From 01c472b4de61b69dd15310ec5592a34077f2900d Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 19:25:08 +0530 Subject: [PATCH 7/8] feat: added unit tests Signed-off-by: Asish Kumar --- pkg/utils/test_data/test_tar/test_src/test.mod | 0 pkg/utils/test_data/test_tar/test_src/test.txt | 0 pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.lock | 0 pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 pkg/utils/test_data/test_tar/test_src/test.mod create mode 100644 pkg/utils/test_data/test_tar/test_src/test.txt create mode 100644 pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.lock create mode 100644 pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.txt diff --git a/pkg/utils/test_data/test_tar/test_src/test.mod b/pkg/utils/test_data/test_tar/test_src/test.mod new file mode 100644 index 00000000..e69de29b diff --git a/pkg/utils/test_data/test_tar/test_src/test.txt b/pkg/utils/test_data/test_tar/test_src/test.txt new file mode 100644 index 00000000..e69de29b diff --git a/pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.lock b/pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.txt b/pkg/utils/test_data/test_tar/test_src/test_tar_dir/test_1.txt new file mode 100644 index 00000000..e69de29b From 0f2e6c517799ae6faa2622b99f951dce9eccbefb Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Sat, 11 May 2024 19:26:09 +0530 Subject: [PATCH 8/8] feat: added unit tests Signed-off-by: Asish Kumar --- pkg/utils/utils_test.go | 66 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 35d77952..d52558eb 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -1,8 +1,12 @@ package utils import ( + "archive/tar" + "io" "os" + "path" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -74,10 +78,68 @@ func TestTarDir(t *testing.T) { if !os.IsNotExist(err) { os.Remove(tarPath) } - emptyArrayOfStrings := []string{} - err = TarDir(filepath.Join(testDir, "test_src"), tarPath, emptyArrayOfStrings, emptyArrayOfStrings) + + testSrcDir := filepath.Join(testDir, "test_src") + + getTarFileNames := func(filePath string) ([]string, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + reader := tar.NewReader(file) + filePaths := []string{} + + for { + header, err := reader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if header.Typeflag != tar.TypeReg { + continue + } + + fullPath := path.Join(header.Name) + fullPath = path.Join(filePath, fullPath) + fullPath = strings.Replace(fullPath, "test.tar", "test_src", 1) + + filePaths = append(filePaths, fullPath) + } + + return filePaths, nil + } + + getNewPattern := func(ex string) string { + return testSrcDir + "/" + ex + } + + err = TarDir(testSrcDir, tarPath, []string{}, []string{}) + assert.Equal(t, err, nil) + _, err = os.Stat(tarPath) + assert.Equal(t, err, nil) + os.Remove(tarPath) + + _ = TarDir(testSrcDir, tarPath, []string{}, []string{"*.mod"}) + fileNames, _ := getTarFileNames(tarPath) + for _, fileName := range fileNames { + flag, _ := filepath.Match(getNewPattern("*.mod"), fileName) + assert.Equal(t, flag, false) + } + _, err = os.Stat(tarPath) assert.Equal(t, err, nil) + os.Remove(tarPath) + _ = TarDir(testSrcDir, tarPath, []string{"*/*.lock", "*.mod"}, []string{}) + fileNames, _ = getTarFileNames(tarPath) + for _, fileName := range fileNames { + flag, _ := filepath.Match(getNewPattern("*/*.lock"), fileName) + assert.Equal(t, flag, true) + } _, err = os.Stat(tarPath) assert.Equal(t, err, nil) os.Remove(tarPath)