Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fixed an issue where dependencies could not be downloaded under the new storage structure #544

Merged
merged 2 commits into from
Nov 15, 2024
Merged

fix: fixed an issue where dependencies could not be downloaded under the new storage structure #544

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5eb17c1
fix: fix missing download when running on new stroage
zong-zhe Nov 15, 2024
23c4211
fix: Add tests under different features
zong-zhe Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 4 additions & 27 deletions pkg/client/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,12 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionNoSumCheck", testAddWithDiffVersionNoSumCheck)
test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionWithSumCheck", testAddWithDiffVersionWithSumCheck)
test.RunTestWithGlobalLock(t, "TestDownloadOci", testDownloadOci)
test.RunTestWithGlobalLock(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
test.RunTestWithGlobalLock(t, "TestAddWithOciDownloader", testAddWithOciDownloader)
test.RunTestWithGlobalLock(t, "TestAddDefaultRegistryDep", testAddDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestUpdateDefaultRegistryDep", testUpdateDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestAddWithNoSumCheck", testAddWithNoSumCheck)
test.RunTestWithGlobalLock(t, "TestAddWithGitCommit", testAddWithGitCommit)
test.RunTestWithGlobalLock(t, "TestDependenciesOrder", testDependenciesOrder)
test.RunTestWithGlobalLock(t, "TestRunInVendor", testRunInVendor)
test.RunTestWithGlobalLock(t, "TestPkgWithInVendorMode", testPkgWithInVendorMode)
test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStrWithPackage", testResolveMetadataInJsonStrWithPackage)
test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStr", testResolveMetadataInJsonStr)
Expand All @@ -63,17 +60,11 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestDownloadGitWithPackage", testDownloadGitWithPackage)
test.RunTestWithGlobalLock(t, "TestModandLockFilesWithGitPackageDownload", testModandLockFilesWithGitPackageDownload)
test.RunTestWithGlobalLock(t, "TestDependencyGraph", testDependencyGraph)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
test.RunTestWithGlobalLock(t, "TestRunWithNoSumCheck", testRunWithGitPackage)
test.RunTestWithGlobalLock(t, "TestRunGit", testRunGit)
test.RunTestWithGlobalLock(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
test.RunTestWithGlobalLock(t, "TestVendorWithGlobalLock", testVendorWithGlobalLock)
test.RunTestWithGlobalLock(t, "TestPull", testPull)
test.RunTestWithGlobalLock(t, "TestPullWithInsecureSkipTLSverify", testPullWithInsecureSkipTLSverify)
test.RunTestWithGlobalLock(t, "TestPullWithModSpec", testPullWithModSpec)
test.RunTestWithGlobalLock(t, "testPullWithOnlySpec", testPullWithOnlySpec)
test.RunTestWithGlobalLock(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)
test.RunTestWithGlobalLock(t, "TestGraph", testGraph)
}

Expand Down Expand Up @@ -1757,7 +1748,7 @@ func testUpdateDefaultRegistryDep(t *testing.T) {
}()
}

func testRunDefaultRegistryDep(t *testing.T) {
func testRunDefaultRegistryDep(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("run_with_default_dep")

pkgWithSumCheckPathModBak := filepath.Join(pkgPath, "kcl.mod.bak")
Expand All @@ -1773,9 +1764,6 @@ func testRunDefaultRegistryDep(t *testing.T) {
err = copy.Copy(pkgWithSumCheckPathModLockBak, pkgWithSumCheckPathModLock)
assert.Equal(t, err, nil)

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

kclPkg, err := kpmcli.LoadPkgFromPath(pkgPath)
assert.Equal(t, err, nil)

Expand Down Expand Up @@ -1996,10 +1984,7 @@ func TestRunLocalWithArgs(t *testing.T) {
}
}

func testRunRemoteWithArgsInvalid(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

func testRunRemoteWithArgsInvalid(t *testing.T, kpmcli *KpmClient) {
logbuf := new(bytes.Buffer)
kpmcli.SetLogWriter(logbuf)

Expand All @@ -2025,10 +2010,8 @@ func testRunRemoteWithArgsInvalid(t *testing.T) {
}
}

func testRunRemoteWithArgs(t *testing.T) {
func testRunRemoteWithArgs(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_options")
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

logbuf := new(bytes.Buffer)
kpmcli.SetLogWriter(logbuf)
Expand Down Expand Up @@ -2091,13 +2074,7 @@ func testRunRemoteWithArgs(t *testing.T) {
}
}

func testRunInVendor(t *testing.T) {
// Create a new kpm client.
kpmcli, err := NewKpmClient()
if err != nil {
t.Fatal(err)
}

func testRunInVendor(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_in_vendor")
workdir := filepath.Join(pkgPath, "pkg")

Expand Down
28 changes: 7 additions & 21 deletions pkg/client/deperated_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,14 @@ import (
"kcl-lang.io/kpm/pkg/utils"
)

func TestRunWithNoSumCheck(t *testing.T) {
func testRunWithNoSumCheck(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_no_sum_check")

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

opts := opt.DefaultCompileOptions()
opts.SetNoSumCheck(true)
opts.SetPkgPath(pkgPath)

_, err = kpmcli.CompileWithOpts(opts)
_, err := kpmcli.CompileWithOpts(opts)
assert.Equal(t, err, nil)
assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "kcl.mod.lock")), false)

Expand All @@ -42,12 +39,9 @@ func TestRunWithNoSumCheck(t *testing.T) {
}()
}

func testRunWithGitPackage(t *testing.T) {
func testRunWithGitPackage(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_git_package")

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

opts := opt.DefaultCompileOptions()
opts.SetPkgPath(pkgPath)

Expand All @@ -63,11 +57,8 @@ func testRunWithGitPackage(t *testing.T) {
}()
}

func testRunWithOciDownloader(t *testing.T) {
kpmCli, err := NewKpmClient()
func testRunWithOciDownloader(t *testing.T, kpmCli *KpmClient) {
path := getTestDir("test_oci_downloader")
assert.Equal(t, err, nil)

kpmCli.DepDownloader = downloader.NewOciDownloader("linux/amd64")

var buf bytes.Buffer
Expand All @@ -84,10 +75,7 @@ func testRunWithOciDownloader(t *testing.T) {
assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!")
}

func testRunGit(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

func testRunGit(t *testing.T, kpmcli *KpmClient) {
testPath := getTestDir("test_run_git")

opts := opt.DefaultCompileOptions()
Expand All @@ -107,15 +95,13 @@ func testRunGit(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(bytes)), utils.RmNewline(string(resultStr)))
}

func testRunOciWithSettingsFile(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
func testRunOciWithSettingsFile(t *testing.T, kpmcli *KpmClient) {
kpmcli.SetLogWriter(nil)
opts := opt.DefaultCompileOptions()
opts.SetEntries([]string{})
opts.Merge(kcl.WithSettings(filepath.Join(".", "test_data", "test_run_oci_with_settings", "kcl.yaml")))
opts.SetHasSettingsYaml(true)
_, err = kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
_, err := kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
assert.Equal(t, err, nil)
}

Expand Down
34 changes: 28 additions & 6 deletions pkg/client/run_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import (
"github.com/otiai10/copy"
"gotest.tools/v3/assert"
"kcl-lang.io/kpm/pkg/downloader"
"kcl-lang.io/kpm/pkg/features"
"kcl-lang.io/kpm/pkg/utils"
)

func testRunWithModSpecVersion(t *testing.T) {
func testRunWithModSpecVersion(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_with_modspec_version")
modbkPath := filepath.Join(pkgPath, "kcl.mod.bk")
modPath := filepath.Join(pkgPath, "kcl.mod")
Expand Down Expand Up @@ -41,11 +42,6 @@ func testRunWithModSpecVersion(t *testing.T) {
}
}()

kpmcli, err := NewKpmClient()
if err != nil {
t.Errorf("Failed to create kpm client: %v", err)
}

res, err := kpmcli.Run(
WithRunSource(
&downloader.Source{
Expand Down Expand Up @@ -83,3 +79,29 @@ func testRunWithModSpecVersion(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(expectedMod)), utils.RmNewline(string(gotMod)))
assert.Equal(t, utils.RmNewline(string(expectedLock)), utils.RmNewline(string(gotLock)))
}

func TestRun(t *testing.T) {
features.Enable(features.SupportNewStorage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
RunTestWithGlobalLockAndKpmCli(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
RunTestWithGlobalLockAndKpmCli(t, "TestRunInVendor", testRunInVendor)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithNoSumCheck", testRunWithNoSumCheck)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithGitPackage", testRunWithGitPackage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunGit", testRunGit)
RunTestWithGlobalLockAndKpmCli(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)

features.Disable(features.SupportNewStorage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
RunTestWithGlobalLockAndKpmCli(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
RunTestWithGlobalLockAndKpmCli(t, "TestRunInVendor", testRunInVendor)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithNoSumCheck", testRunWithNoSumCheck)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithGitPackage", testRunWithGitPackage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunGit", testRunGit)
RunTestWithGlobalLockAndKpmCli(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)
}
79 changes: 66 additions & 13 deletions pkg/downloader/downloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,39 @@ func (d *OciDownloader) Download(opts *DownloadOptions) error {
return err
}
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf(
"downloading '%s:%s' from '%s/%s:%s'",
ociSource.Repo, ociSource.Tag, ociSource.Reg, ociSource.Repo, ociSource.Tag,
),
opts.LogWriter,
)

err = ociCli.Pull(localPath, ociSource.Tag)
if err != nil {
return err
}
tarPath, err := utils.FindPkgArchive(localPath)
if err != nil {
return err
}
if utils.IsTar(tarPath) {
err = utils.UnTarDir(tarPath, localPath)
} else {
err = utils.ExtractTarball(tarPath, localPath)
}
if err != nil {
return fmt.Errorf("failed to untar the kcl package tar from '%s' into '%s'", tarPath, localPath)
}

// After untar the downloaded kcl package tar file, remove the tar file.
if utils.DirExists(tarPath) {
rmErr := os.Remove(tarPath)
if rmErr != nil {
return fmt.Errorf("failed to remove the downloaded kcl package tar file '%s'", tarPath)
}
}
}
} else {
reporter.ReportMsgTo(
Expand Down Expand Up @@ -498,6 +531,19 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
git.WithTag(gitSource.Tag),
}

var msg string
if len(opts.Source.Git.Tag) != 0 {
msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
}

if len(opts.Source.Git.Commit) != 0 {
msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
}

if len(opts.Source.Git.Branch) != 0 {
msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
}

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
cacheFullPath := opts.CachePath
Expand Down Expand Up @@ -525,6 +571,10 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
return err
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
)
// If not, clone the bare repository from the remote git repository, update the cache.
if utils.DirExists(cacheFullPath) {
err = os.Remove(cacheFullPath)
Expand Down Expand Up @@ -558,21 +608,24 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
}
}
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
)
// If the cache is disabled, clone the repository from the remote git repository.
_, err := git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(opts.LocalPath),
)...,
)
if err != nil {
return err
}
}
} else {
var msg string
if len(opts.Source.Git.Tag) != 0 {
msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
}

if len(opts.Source.Git.Commit) != 0 {
msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
}

if len(opts.Source.Git.Branch) != 0 {
msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
}

reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
Expand Down