From e8ddeed45de77bbc6523194b5fe2b1a9bb10aa35 Mon Sep 17 00:00:00 2001 From: Shuheng Liu Date: Fri, 10 Jan 2025 12:07:46 -0500 Subject: [PATCH 1/4] fix: convert semver to valid OCI tags when publishing to ORAS --- src/cls/IPM/Repo/Oras/PublishService.cls | 24 +++++++++++++----------- src/inc/IPM/Common.inc | 6 +++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/cls/IPM/Repo/Oras/PublishService.cls b/src/cls/IPM/Repo/Oras/PublishService.cls index 1889cc5a..4574c28c 100644 --- a/src/cls/IPM/Repo/Oras/PublishService.cls +++ b/src/cls/IPM/Repo/Oras/PublishService.cls @@ -1,3 +1,5 @@ +Include %IPM.Common + Class %IPM.Repo.Oras.PublishService Extends (%IPM.Repo.Oras.PackageService, %IPM.Repo.IPublishService) { @@ -6,7 +8,7 @@ Method PublishModule(pModule As %IPM.Repo.Remote.ModuleInfo) As %Status Set status = $$$OK Try { Set repo = pModule.Name - Set tag = pModule.VersionString + Set tag = $$$Semver2Tag(pModule.VersionString) #; Use a temp directory Set tDir = $$$FileTempDir @@ -85,7 +87,7 @@ ClassMethod Push(registry As %String, package As %String, namespace As %String, { #; Remove the tag from the package name if it exists and add it to tags Set repo = $piece(package, ":", 1) - Set tag = $piece(package, ":", 2) + Set tag = $$$Semver2Tag($piece(package, ":", 2)) If tag '= "" { Set tags = tags _ "," _ tag } If tags = "" { $$$ThrowStatus($$$ERROR($$$GeneralError,"Must specify version.")) @@ -108,31 +110,31 @@ ClassMethod PushPy(registry As %String, package As %String, namespace As %String import iris import os, sys import json + import re # Get all files in the directory and fully specify path - files = os.listdir(directoryPath) - result = map(lambda x: directoryPath + x, files) - file_paths = list(result) - + file_paths = [os.path.join(directoryPath, f) for f in os.listdir(directoryPath)] client = iris.cls("%IPM.Repo.Oras.PackageService").GetClient(registry, username, password, token, tokenAuthMethod) + regex = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$") # Push once for each tag for tag in tags.split(","): - if tag == "": - continue + if not regex.search(tag): + raise ValueError(f"Invalid OCI tag: {tag}") # Annotations are manifest.xml and other metadata manifest_annotations = json.loads(metadata) + # TODO write a context manager for stdout/stderr redirection # Suppress console output sys.stdout = open(os.devnull, "w") - target = iris.cls("%IPM.Repo.Oras.PackageService").GetAPITarget(registry, package, namespace) + ":" + tag - try: + target = iris.cls("%IPM.Repo.Oras.PackageService").GetAPITarget(registry, package, namespace) + ":" + tag res = client.push(files=file_paths, target=target, disable_path_validation=True, manifest_annotations=manifest_annotations) except Exception as e: - print("Error: ", repr(e)) + print("Error: ", repr(e), file=sys.stderr) + sys.stdout = sys.__stdout__ raise e # Reenable console output diff --git a/src/inc/IPM/Common.inc b/src/inc/IPM/Common.inc index 06dce37d..f30b85de 100644 --- a/src/inc/IPM/Common.inc +++ b/src/inc/IPM/Common.inc @@ -66,4 +66,8 @@ ROUTINE %IPM.Common [Type=INC] #def1arg ENDTAG ##Expression($$$ENDTAGQ) #; Variable to mark deprecation warning has been shown in the current process -#define DeprecationWarned %IPMModuleDeprecatedResource \ No newline at end of file +#define DeprecationWarned %IPMModuleDeprecatedResource + +#; Convert module version from/to OCI tag +#def1arg Semver2Tag(%semver) $Replace(%semver,"+","_") +#def1arg Tag2Semver(%tag) $Replace(%tag,"_","+") \ No newline at end of file From 3d28451080cc501e51f082ecc89afaf5c1bd6762 Mon Sep 17 00:00:00 2001 From: Shuheng Liu Date: Fri, 10 Jan 2025 13:16:13 -0500 Subject: [PATCH 2/4] fix: convert semver to/from OCI tag when installing from ORAS registry --- src/cls/IPM/Repo/Oras/PackageService.cls | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/cls/IPM/Repo/Oras/PackageService.cls b/src/cls/IPM/Repo/Oras/PackageService.cls index 2b7e1b76..78860b58 100644 --- a/src/cls/IPM/Repo/Oras/PackageService.cls +++ b/src/cls/IPM/Repo/Oras/PackageService.cls @@ -32,7 +32,7 @@ Method IsAvailable() As %Boolean Method GetModule(pModuleReference As %IPM.Storage.ModuleInfo, Output AsArchive As %Boolean = 0) As %Stream.Object { - Set name = pModuleReference.Name _ ":" _ pModuleReference.VersionString + Set name = pModuleReference.Name _ ":" _ $$$Semver2Tag(pModuleReference.VersionString) Set status = ..Pull(..Location, name, ..Namespace, ..Username, ..Password, ..Token, ..TokenAuthMethod, .stream) $$$ThrowOnError(status) #; module is pulled as a .tgz file @@ -42,13 +42,13 @@ Method GetModule(pModuleReference As %IPM.Storage.ModuleInfo, Output AsArchive A Method GetModuleManifest(pModuleReference As %IPM.Storage.ModuleInfo) As %Stream.Object { - Set name = pModuleReference.Name _ ":" _ pModuleReference.VersionString + Set name = pModuleReference.Name _ ":" _ $$$Semver2Tag(pModuleReference.VersionString) Return ..GetModuleXMLPy(..Location, name, ..Namespace, ..Username, ..Password, ..Token, ..TokenAuthMethod) } Method HasModule(pModuleReference As %IPM.Storage.ModuleInfo) As %Boolean { - Set name = pModuleReference.Name _ ":" _ pModuleReference.VersionString + Set name = pModuleReference.Name _ ":" _ $$$Semver2Tag(pModuleReference.VersionString) #; Get ORAS client Set client = ..GetClient(..Location, ..Username, ..Password, ..Token, ..TokenAuthMethod) @@ -238,7 +238,7 @@ ClassMethod PullOras(registry As %String, package As %String, namespace As %Stri { Set pkgTag = ..GetPackageVersionPy(registry, package, namespace, username, password, token, tokenAuthMethod, 1) Set pkg = $PIECE(pkgTag, ",", 1) - Set tag = $PIECE(pkgTag, ",", 2) + Set tag = $$$Semver2Tag($PIECE(pkgTag, ",", 2)) Set target = ..GetAPITarget(registry, pkg, namespace) _ ":" _ tag Set client = ..GetClient(registry, username, password, token, tokenAuthMethod) @@ -295,9 +295,9 @@ ClassMethod GetPackageMetadataPy(registry As %String, package As %String, namesp /// Given a package name, either parses out the tag or determines the latest tag /// Params: /// package : is either in the form : or -/// in the latter case, returns latest tag -/// asString : 0 => returns as python tuple of (package, tag) -/// 1 => returns as string of "," +/// in the latter case, returns semver corresponding to the latest tag +/// asString : 0 => returns as python tuple of (package, semver) +/// 1 => returns as string of "," ClassMethod GetPackageVersionPy(registry As %String, package As %String, namespace As %String, username As %String, password As %String, token As %String, tokenAuthMethod As %String, asString As %Boolean = 0) [ Language = python ] { import iris @@ -307,10 +307,12 @@ ClassMethod GetPackageVersionPy(registry As %String, package As %String, namespa else: pkg = package tag = iris.cls("%IPM.Repo.Oras.PackageService").GetLatestTagPy(registry, pkg, namespace, username, password, token, tokenAuthMethod, asString ) + + semver = tag.replace("_", "+") if asString: - return pkg + "," + tag + return pkg + "," + semver else: - return (pkg, tag) + return (pkg, semver) } /// Lists all of the modules and their latest versions in the package From ab8a5b41b805b46cd96e53904ce473beb99fbb13 Mon Sep 17 00:00:00 2001 From: Shuheng Liu Date: Fri, 10 Jan 2025 13:40:23 -0500 Subject: [PATCH 3/4] test: add test cases for oras tag conversion --- .../Test/PM/Integration/OrasTag.cls | 35 +++++++++++++++++++ .../PM/Integration/_data/oras-tag/module.xml | 12 +++++++ .../_data/oras-tag/src/OrasTag/Main.cls | 9 +++++ 3 files changed, 56 insertions(+) create mode 100644 tests/integration_tests/Test/PM/Integration/OrasTag.cls create mode 100644 tests/integration_tests/Test/PM/Integration/_data/oras-tag/module.xml create mode 100644 tests/integration_tests/Test/PM/Integration/_data/oras-tag/src/OrasTag/Main.cls diff --git a/tests/integration_tests/Test/PM/Integration/OrasTag.cls b/tests/integration_tests/Test/PM/Integration/OrasTag.cls new file mode 100644 index 00000000..fef9f10a --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/OrasTag.cls @@ -0,0 +1,35 @@ +Class Test.PM.Integration.OrasTag Extends Test.PM.Integration.Base +{ + +Parameter TargetModuleName As STRING = "oras-tag"; + +Method TestOrasTagConversion() +{ + Set tModuleDir = ..GetModuleDir(..#TargetModuleName) + + Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir) + Do $$$AssertStatusOK(tSC,"Loaded module successfully") + + Set tSC = ##class(%IPM.Main).Shell("repo -delete-all") + Do $$$AssertStatusOK(tSC,"Deleted repos successfully") + + Set tSC = ##class(%IPM.Main).Shell("repo -o -name oras -url http://oras:5000 -publish 1") + Do $$$AssertStatusOK(tSC,"Set up oras module successfully") + + Set tSC = ##class(%IPM.Main).Shell("publish oras-tag -r oras -verbose") + Do $$$AssertStatusOK(tSC,"Published module successfully") + + Set tSC = ##class(%IPM.Main).Shell("uninstall oras-tag") + Do $$$AssertStatusOK(tSC,"Uninstalled module successfully") + + Set tSC = ##class(%IPM.Main).Shell("install oras-tag") + Do $$$AssertStatusOK(tSC,"Installed module from ORAS registry successfully") + + Set tSC = ##class(%IPM.Main).Shell("repo -delete-all") + Do $$$AssertStatusOK(tSC,"Deleted repos successfully") + + Set tSC = ##class(%IPM.Main).Shell("repo -reset-defaults") + Do $$$AssertStatusOK(tSC,"Reset repos to default successfully") +} + +} diff --git a/tests/integration_tests/Test/PM/Integration/_data/oras-tag/module.xml b/tests/integration_tests/Test/PM/Integration/_data/oras-tag/module.xml new file mode 100644 index 00000000..70b33d08 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/oras-tag/module.xml @@ -0,0 +1,12 @@ + + + + + oras-tag + 0.0.1-beta.1+build + module + src + + + + \ No newline at end of file diff --git a/tests/integration_tests/Test/PM/Integration/_data/oras-tag/src/OrasTag/Main.cls b/tests/integration_tests/Test/PM/Integration/_data/oras-tag/src/OrasTag/Main.cls new file mode 100644 index 00000000..a8080da3 --- /dev/null +++ b/tests/integration_tests/Test/PM/Integration/_data/oras-tag/src/OrasTag/Main.cls @@ -0,0 +1,9 @@ +Class OrasTag.Main +{ + +ClassMethod Main() +{ + Write "This is OrasTag.main" +} + +} From 9764c0ba83d251f0a8d15ec3e25712a49064bfff Mon Sep 17 00:00:00 2001 From: Shuheng Liu Date: Fri, 10 Jan 2025 13:42:28 -0500 Subject: [PATCH 4/4] chore: update change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e697b4e7..2192d60e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - #474: When loading a .tgz/.tar.gz package, automatically locate the top-most module.xml in case there is nested directory structure (e.g., GitHub releases) - #635: When calling the "package" command, the directory is now normalized to include trailing slash (or backslash). - #696: Fix a bug that caused error status to be ignored when publishing a module. +- #700: Fix a bug due to incompatible conventions between SemVer and OCI tags ### Security -