feat: run remote repo in kpm run (#251)

* feat: run remote repo in kpm run Signed-off-by: Akash Kumar <meakash7902@gmail.com> * address review comments Signed-off-by: Akash Kumar <meakash7902@gmail.com> * test: Add unit test Signed-off-by: Akash Kumar <meakash7902@gmail.com> --------- Signed-off-by: Akash Kumar <meakash7902@gmail.com>
kcl-lang · Feb 18, 2024 · f67db1b · f67db1b
1 parent bd65b2a
commit f67db1b
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 2 deletions.
diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -439,6 +439,33 @@ func (c *KpmClient) CompileTarPkg(tarPath string, opts *opt.CompileOptions) (*kc
 	return c.CompileWithOpts(opts)
 }
 
+// CompileGitPkg will compile the kcl package from the git url.
+func (c *KpmClient) CompileGitPkg(gitOpts *git.CloneOptions, compileOpts *opt.CompileOptions) (*kcl.KCLResultList, error) {
+	// 1. Create the temporary directory to pull the tar.
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.")
+	}
+	// clean the temp dir.
+	defer os.RemoveAll(tmpDir)
+
+	// 2. clone the git repo
+	_, err = git.CloneWithOpts(
+		git.WithCommit(gitOpts.Commit),
+		git.WithBranch(gitOpts.Branch),
+		git.WithTag(gitOpts.Tag),
+		git.WithRepoURL(gitOpts.RepoURL),
+		git.WithLocalPath(tmpDir),
+	)
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.FailedGetPkg, err, "failed to get the git repository")
+	}
+
+	compileOpts.SetPkgPath(tmpDir)
+
+	return c.CompileWithOpts(compileOpts)
+}
+
 // CompileOciPkg will compile the kcl package from the OCI reference or url.
 func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) {
 	ociOpts, err := c.ParseOciOptionFromString(ociSource, version)

diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
 	"kcl-lang.io/kpm/pkg/env"
+	"kcl-lang.io/kpm/pkg/git"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
 	"kcl-lang.io/kpm/pkg/runner"
@@ -844,6 +845,29 @@ func TestRunWithNoSumCheck(t *testing.T) {
 	}()
 }
 
+func TestRemoteRun(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	opts := opt.DefaultCompileOptions()
+	gitOpts := git.NewCloneOptions("https://github.com/KusionStack/catalog", "", "0.1.2", "", "", nil)
+
+	opts.SetEntries([]string{"models/samples/hellocollaset/prod/main.k"})
+	result, err := kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"hellocollaset\": {\"workload\": {\"containers\": {\"nginx\": {\"image\": \"nginx:v2\"}}}}}]")
+
+	opts.SetEntries([]string{"models/samples/pgadmin/base/base.k"})
+	result, err = kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"pgadmin\": {\"workload\": {\"containers\": {\"pgadmin\": {\"image\": \"dpage/pgadmin4:latest\", \"env\": {\"PGADMIN_DEFAULT_EMAIL\": \"admin@admin.com\", \"PGADMIN_DEFAULT_PASSWORD\": \"secret://pgadmin-secret/pgadmin-default-password\", \"PGADMIN_PORT\": \"80\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"secrets\": {\"pgadmin-secret\": {\"type\": \"opaque\", \"data\": {\"pgadmin-default-password\": \"*******\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"pgadmin\": {\"type\": \"cloud\", \"version\": \"14.0\"}}}}]")
+
+	opts.SetEntries([]string{"models/samples/wordpress/prod/main.k"})
+	result, err = kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"wordpress\": {\"workload\": {\"containers\": {\"wordpress\": {\"image\": \"wordpress:6.3\", \"env\": {\"WORDPRESS_DB_HOST\": \"$(KUSION_DB_HOST_WORDPRESS)\", \"WORDPRESS_DB_USER\": \"$(KUSION_DB_USERNAME_WORDPRESS)\", \"WORDPRESS_DB_PASSWORD\": \"$(KUSION_DB_PASSWORD_WORDPRESS)\", \"WORDPRESS_DB_NAME\": \"mysql\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"wordpress\": {\"type\": \"cloud\", \"version\": \"8.0\"}}}}]")
+}
+
 func TestUpdateWithNoSumCheck(t *testing.T) {
 	pkgPath := getTestDir("test_update_no_sum_check")
 	kpmcli, err := NewKpmClient()

diff --git a/pkg/cmd/cmd_run.go b/pkg/cmd/cmd_run.go
@@ -12,6 +12,7 @@ import (
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/api"
 	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/git"
 	"kcl-lang.io/kpm/pkg/opt"
 	"kcl-lang.io/kpm/pkg/reporter"
 	"kcl-lang.io/kpm/pkg/runner"
@@ -110,7 +111,7 @@ func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error {
 		return errEvent
 	}
 
-	// 'kpm run' compile the current package undor '$pwd'.
+	// 'kpm run' compile the current package under '$pwd'.
 	if runEntry.IsEmpty() {
 		pwd, err := os.Getwd()
 		kclOpts.SetPkgPath(pwd)
@@ -140,8 +141,12 @@ func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error {
 				compileResult, err = kpmcli.CompileWithOpts(kclOpts)
 			}
 		} else if runEntry.IsTar() {
-			// 'kpm run' compile the package from the kcl pakcage tar.
+			// 'kpm run' compile the package from the kcl package tar.
 			compileResult, err = kpmcli.CompileTarPkg(runEntry.PackageSource(), kclOpts)
+		} else if runEntry.IsGit() {
+			gitOpts := git.NewCloneOptions(runEntry.PackageSource(), "", c.String(FLAG_TAG), "", "", nil)
+			// 'kpm run' compile the package from the git url
+			compileResult, err = kpmcli.CompileGitPkg(gitOpts, kclOpts)
 		} else {
 			// 'kpm run' compile the package from the OCI reference or url.
 			compileResult, err = kpmcli.CompileOciPkg(runEntry.PackageSource(), c.String(FLAG_TAG), kclOpts)

diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
@@ -9,6 +9,7 @@ const (
 	UrlEntry                             = "url"
 	RefEntry                             = "ref"
 	TarEntry                             = "tar"
+	GitEntry                             = "git"
 	KCL_MOD                              = "kcl.mod"
 	OCI_SEPARATOR                        = ":"
 	KCL_PKG_TAR                          = "*.tar"

diff --git a/pkg/git/git.go b/pkg/git/git.go
@@ -24,6 +24,17 @@ type CloneOptions struct {
 // CloneOption is a function that modifies CloneOptions
 type CloneOption func(*CloneOptions)
 
+func NewCloneOptions(repoUrl, commit, tag, branch, localpath string, Writer io.Writer) *CloneOptions {
+	return &CloneOptions{
+		RepoURL:   repoUrl,
+		Commit:    commit,
+		Tag:       tag,
+		Branch:    branch,
+		LocalPath: localpath,
+		Writer:    Writer,
+	}
+}
+
 // WithRepoURL sets the repo URL for CloneOptions
 func WithRepoURL(repoURL string) CloneOption {
 	return func(o *CloneOptions) {

diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go
@@ -24,6 +24,16 @@ func TestWithGitOptions(t *testing.T) {
 	assert.Equal(t, cloneOpts.Writer, nil)
 }
 
+func TestNewCloneOptions(t *testing.T) {
+	cloneOpts := NewCloneOptions("https://github.com/kcl-lang/kcl", "", "v1.0.0", "", "", nil)
+	assert.Equal(t, cloneOpts.RepoURL, "https://github.com/kcl-lang/kcl")
+	assert.Equal(t, cloneOpts.Tag, "v1.0.0")
+	assert.Equal(t, cloneOpts.Commit, "")
+	assert.Equal(t, cloneOpts.Branch, "")
+	assert.Equal(t, cloneOpts.LocalPath, "")
+	assert.Equal(t, cloneOpts.Writer, nil)
+}
+
 func TestValidateGitOptions(t *testing.T) {
 	cloneOpts := &CloneOptions{}
 	WithBranch("test_branch")(cloneOpts)

diff --git a/pkg/runner/entry.go b/pkg/runner/entry.go
@@ -67,6 +67,10 @@ func (e *Entry) IsTar() bool {
 	return e.kind == constants.TarEntry
 }
 
+func (e *Entry) IsGit() bool {
+	return e.kind == constants.GitEntry
+}
+
 // IsEmpty will return true if the entry is empty.
 func (e *Entry) IsEmpty() bool {
 	return len(e.packageSource) == 0
@@ -166,6 +170,8 @@ func GetSourceKindFrom(source string) EntryKind {
 		return constants.FileEntry
 	} else if utils.IsTar(source) {
 		return constants.TarEntry
+	} else if utils.IsGitRepoUrl(source) {
+		return constants.GitEntry
 	} else if utils.IsURL(source) {
 		return constants.UrlEntry
 	} else if utils.IsRef(source) {

diff --git a/pkg/runner/entry_test.go b/pkg/runner/entry_test.go
@@ -44,6 +44,7 @@ func TestGetSourceKindFrom(t *testing.T) {
 	assert.Equal(t, string(GetSourceKindFrom("./testdata_external/external/main.k")), constants.FileEntry)
 	assert.Equal(t, string(GetSourceKindFrom("main.tar")), constants.TarEntry)
 	assert.Equal(t, string(GetSourceKindFrom("oci://test_url")), constants.UrlEntry)
+	assert.Equal(t, string(GetSourceKindFrom("https://github.com/test_org/test")), constants.GitEntry)
 	assert.Equal(t, string(GetSourceKindFrom("test_ref:0.0.1")), constants.RefEntry)
 	assert.Equal(t, string(GetSourceKindFrom("invalid input")), "")
 }

diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	goerrors "errors"
@@ -378,6 +379,12 @@ func IsURL(str string) bool {
 	return err == nil && u.Scheme != "" && u.Host != ""
 }
 
+// IsGitRepoUrl will check whether the string 'str' is a git repo url
+func IsGitRepoUrl(str string) bool {
+	r := regexp.MustCompile(`((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?`)
+	return r.MatchString(str)
+}
+
 // IsRef will check whether the string 'str' is a reference.
 func IsRef(str string) bool {
 	_, err := reference.ParseNormalizedNamed(str)

diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go
@@ -138,6 +138,40 @@ func TestIsUrl(t *testing.T) {
 	assert.Equal(t, IsURL("https://"), false)
 }
 
+func TestIsGitRepoUrl(t *testing.T) {
+	assert.Equal(t, IsGitRepoUrl("invalid url"), false)
+	assert.Equal(t, IsGitRepoUrl("ftp://github.com/user/project.git"), false)
+	assert.Equal(t, IsGitRepoUrl("file:///path/to/repo.git/"), false)
+	assert.Equal(t, IsGitRepoUrl("file://~/path/to/repo.git/"), false)
+	assert.Equal(t, IsGitRepoUrl("path/to/repo.git/"), false)
+	assert.Equal(t, IsGitRepoUrl("~/path/to/repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("rsync://host.xz/path/to/repo.git/"), false)
+	assert.Equal(t, IsGitRepoUrl("host.xz:path/to/repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("user@host.xz:path/to/repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("C:\\path\\to\\repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("/path/to/repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("./path/to/repo.git"), false)
+	assert.Equal(t, IsGitRepoUrl("oci://host.xz/path/to/repo.git/"), false)
+	assert.Equal(t, IsGitRepoUrl("https://github.com/user/project"), true)
+	assert.Equal(t, IsGitRepoUrl("git@github.com:user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("https://github.com/user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("https://github.com/user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("git@192.168.101.127:user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("https://192.168.101.127/user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("http://192.168.101.127/user/project.git"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz:port/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://host.xz:port/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://host.xz/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/~user/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://host.xz/~user/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/~/path/to/repo.git"), true)
+	assert.Equal(t, IsGitRepoUrl("git://host.xz/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("http://host.xz/path/to/repo.git/"), true)
+	assert.Equal(t, IsGitRepoUrl("https://host.xz/path/to/repo.git/"), true)
+}
+
 func TestIsRef(t *testing.T) {
 	assert.Equal(t, IsRef("invalid ref"), false)
 	assert.Equal(t, IsRef("ghcr.io/xxx/xxx"), true)