fix: migrate standard repos, allow partial migration of packages

Plus testing for repo migration, and a comment on multiple file extension support in Default.Document.

src/cls/IPM/ResourceProcessor/Default/Document.cls

@@ -126,6 +126,10 @@ Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boo
             Set tFileExtension = ""
             Set tResourcePath = ##class(%File).NormalizeFilename(tResourceDirectory _ tSubDirectory _ $Translate(tName, ..FilenameTranslateIdentifier, ..FilenameTranslateAssociator))
             If '..LoadAsDirectory {
+              // For user document types (generally), dynamilly support both the resource and filename extension
+              // e.g., for lookup tables, both .xml and .lut are supported regardless of configuration in module.xml.
+              // Generally, FilenameExtension SHOULD be specified (for the sake of embedded source control integration if nothing else),
+              // but this makes IPM less picky about reasonable things users might do.
               Set tItemFileExtension = $$$lcase("." _ $Piece(..ResourceReference.Name, ".", *))
               Set tFileExtension =  $$$lcase("." _ ..FilenameExtension)
               For tFileExtension = tFileExtension,tItemFileExtension {

src/cls/IPM/Utils/Migration.cls

@@ -14,73 +14,97 @@ ClassMethod RunAll(verbose As %Boolean = 1) As %Status
 
 ClassMethod MigrateZPMToIPM(verbose As %Boolean = 1)
 {
-    If '$$$comClassDefined("%ZPM.PackageManager.Developer.Module") {
+    Set oldTopPackage = "%ZPM.PackageManager."
+    If $Order(^oddCOM(oldTopPackage)) '[ oldTopPackage {
         Write:verbose !,"Older IPM version not found; nothing to migrate.",!
         Quit
     }
 
     Set sc = $$$OK
-    Set initTLevel = $TLevel
+    Set successPackageList = ""
+    Set failedPackageList = ""
+
     // Intentionally runs for current namespace only
     Try {
-        TSTART
-
         // ^ZPM.Dev.ModuleD -> ^IPM.Storage.ModuleD
         Set oldId = ""
         For {
             Set oldId = $order(^ZPM.Dev.ModuleD(oldId),1,moduleData)
             Quit:oldId=""
-
-            Set name = $ListGet(moduleData,2)
-            If ##class(%IPM.Storage.Module).NameExists(name) {
-                Write:verbose !,"Package already found in new storage: ",name
-                Continue
+            Set moduleName = $ListGet(moduleData,2)
+            Set oneSC = ..MigrateOnePackageToIPM(oldId, moduleName, verbose)
+            If $$$ISOK(oneSC) {
+                Set successPackageList = successPackageList _ $ListBuild(moduleName)
+            } Else {
+                Set failedPackageList = failedPackageList _ $ListBuild(moduleName)
             }
+            Set sc = $$$ADDSC(sc,oneSC)
+        }
 
-            Write:verbose !,"Migrating storage for ",name," ... "
-
-            // Bring everything over - storage definitions are fortunately compatible.
-            Kill moduleData
-            Merge moduleData = ^ZPM.Dev.ModuleD(oldId)
+        $$$ThrowOnError(##class(%IPM.Storage.Module).%BuildIndices(,1,1))
+        $$$ThrowOnError(..MigrateRepoDefinitions())
+        $$$ThrowOnError(sc)
 
-            // In ^IPM.Storage.ModuleD(<ID>,"Defaults",n) = $lb($lb(<data>),classname), map classname
-            // to equivalent - %ZPM.PackageManager.Developer.ModuleSetting -> %IPM.Storage.ModuleSetting
-            Set defaultKey = ""
-            For {
-                Set defaultKey = $Order(moduleData("Defaults",defaultKey),1,defaultData)
-                Quit:defaultKey=""
-                Set moduleData("Defaults",defaultKey) = $ListBuild($ListGet(defaultData,1),$Replace($ListGet(defaultData,2),"%ZPM.PackageManager.Developer.ModuleSetting","%IPM.Storage.ModuleSetting"))
-            }
+        Write:verbose !,"Migrated successfully: ",$ListToString(successPackageList)
+    } Catch e {
+        Set sc = e.AsStatus()
+        // Show these messages even in verbose mode.
+        Write !,"IPM data migration was not fully successful. You may be able to reinstall the impacted packages to correct the issues."
+        Write !,"Migrated successfully: ",$ListToString(successPackageList)
+        Write !,"Failed to migrate: ",$ListToString(failedPackageList)
+        Write !,$System.Status.GetErrorText(sc)
+    }
+    Write:verbose !
+    $$$ThrowOnError(sc)
+}
 
-            Set newId = $Increment(^IPM.Storage.ModuleD)
-            Merge ^IPM.Storage.ModuleD(newId) = moduleData
+ClassMethod MigrateOnePackageToIPM(oldId As %Integer, name As %String, verbose As %Boolean = 1)
+{
+    Set sc = $$$OK
+    Set initTLevel = $TLevel
+    Try {
+        If ##class(%IPM.Storage.Module).NameExists(name) {
+            Write:verbose !,"Package already found in new storage: ",name
+            Quit
+        }
 
-            // Make sure loading the object works
-            Set newObj = ##class(%IPM.Storage.Module).%OpenId(newId,,.sc)
-            $$$ThrowOnError(sc)
+        Write:verbose !,"Migrating storage for ",name," ... "
 
-            // Save object to validate
-            $$$ThrowOnError(newObj.%Save())
+        // Bring everything over - storage definitions are fortunately compatible.
+        TSTART
+        Kill moduleData
+        Merge moduleData = ^ZPM.Dev.ModuleD(oldId)
 
-            Write:verbose "done."
+        // In ^IPM.Storage.ModuleD(<ID>,"Defaults",n) = $lb($lb(<data>),classname), map classname
+        // to equivalent - %ZPM.PackageManager.Developer.ModuleSetting -> %IPM.Storage.ModuleSetting
+        Set defaultKey = ""
+        For {
+            Set defaultKey = $Order(moduleData("Defaults",defaultKey),1,defaultData)
+            Quit:defaultKey=""
+            Set moduleData("Defaults",defaultKey) = $ListBuild($ListGet(defaultData,1),$Replace($ListGet(defaultData,2),"%ZPM.PackageManager.Developer.ModuleSetting","%IPM.Storage.ModuleSetting"))
         }
 
-        $$$ThrowOnError(##class(%IPM.Storage.Module).%BuildIndices(,1,1))
+        Set newId = $Increment(^IPM.Storage.ModuleD)
+        Merge ^IPM.Storage.ModuleD(newId) = moduleData
+
+        // Make sure loading the object works
+        Set newObj = ##class(%IPM.Storage.Module).%OpenId(newId,,.sc)
+        $$$ThrowOnError(sc)
+
+        // Save object to validate
+        $$$ThrowOnError(newObj.%Save())
+
+        Write:verbose "done."
 
         // Migrate %ZPM.PackageManager.Developer.Document.ModuleStream to %IPM.StudioDocument.ModuleStream
         // ^ZPM.Dev.ModuleDocD -> ^IPM.StudioDoc.ModuleStreamD
         // Uses objects for the actual data migration to more cleanly handle the stream.
-        Set oldId = ""
-        For {
-            Set oldId = $order(^ZPM.Dev.ModuleDocD(oldId),1,moduleDocData)
-            Quit:oldId=""
-
-            Set name = $ListGet(moduleDocData,2)
+        Do {
             If ##class(%IPM.StudioDocument.ModuleStream).NameExists(name) {
                 Write:verbose !,"Document already found in new storage: ",name
-                Continue
+                Quit
             }
-            Set oldObj = ##class(%ZPM.PackageManager.Developer.Document.ModuleStream).%OpenId(oldId,,.sc)
+            Set oldObj = ##class(%ZPM.PackageManager.Developer.Document.ModuleStream).NameOpen(name,,.sc)
             $$$ThrowOnError(sc)
             Set newObj = ##class(%IPM.StudioDocument.ModuleStream).%New()
             Set newObj.Name = oldObj.Name
@@ -89,20 +113,78 @@ ClassMethod MigrateZPMToIPM(verbose As %Boolean = 1)
             $$$ThrowOnError(newObj.Contents.CopyFrom(oldObj.Contents))
             $$$ThrowOnError(newObj.%Save())
 
-            Write:verbose !,"Migrated document stream data for "_newObj.Name
+            Write:verbose !,"Migrated document stream - "_newObj.Name_".ZPM"
+        } While 0
+
+        TCOMMIT
+    } Catch e {
+        Set sc = e.AsStatus()
+    }
+    While $TLevel > initTLevel {
+        TROLLBACK 1
+    }
+    Quit sc
+}
+
+ClassMethod MigrateRepoDefinitions(verbose As %Boolean = 1) As %Status
+{
+    Set initTLevel = $TLevel
+    Try {
+        TSTART
+        Set key = ""
+        For {
+            Set key = $Order(^%ZPM.Client.ServerDefD(key),1,repoData)
+            Quit:key=""
+
+            Set repoName = $ListGet(repoData,2)
+            Do ..MigrateOneRepo(key, repoName, verbose)
         }
 
-        TCOMMIT        
+        $$$ThrowOnError(##class(%IPM.Repo.Definition).%BuildIndices(,1,1))
+        $$$ThrowOnError(##class(%IPM.Repo.Remote.Definition).%BuildIndices(,1,1))
+        TCOMMIT
     } Catch e {
         Set sc = e.AsStatus()
-        Write:verbose !,"IPM data migration unsuccessful."
-        Write:verbose !,$System.Status.GetErrorText(sc)
     }
-    Write:verbose !
     While $TLevel > initTLevel {
         TROLLBACK 1
     }
+    Quit sc
+}
+
+ClassMethod MigrateOneRepo(oldId As %String, name As %String, verbose As %Boolean)
+{
+    If ##class(%IPM.Repo.Remote.Definition).ServerDefinitionKeyExists(name) {
+        Write:verbose !,"Skipping migrating repo "_name_" - already exists."
+        Quit
+    }
+    Merge data = ^%ZPM.Client.ServerDefD(oldId)
+
+    Set class = $ListGet(data)
+    If (class '= "~%ZPM.PackageManager.Client.RemoteServerDefinition~") {
+        Write:verbose !,"Skipping migrating repo "_name_" - unsupported repo type."
+        Quit
+    }
+
+    Write:verbose !,"Migrating repo "_name_"... "
+
+    Set $List(data,1) = "~%IPM.Repo.Remote.Definition~"
+    Set data("RepoDefinition") = data
+    Merge data("RemoteRepoDefinition") = data("RemoteServerDefinition")
+    ZKill data
+    ZKill data("RemoteServerDefinition")
+
+    Set newId = $Increment(^IPM.Repo.DefinitionD)
+    Merge ^IPM.Repo.DefinitionD(newId) = data
+
+    // Make sure loading/saving the object works
+    Set newObj = ##class(%IPM.Storage.Module).%OpenId(newId,,.sc)
     $$$ThrowOnError(sc)
+
+    // Save object to validate
+    $$$ThrowOnError(newObj.%Save())
+
+    Write:verbose !,"done."
 }
 
 }

tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/ModuleStorage.cls

@@ -19,4 +19,15 @@ Method TestZPM()
     Do $$$AssertTrue(##class(%IPM.StudioDocument.Module).Exists("zpm.ZPM"))
 }
 
+Method TestRepos()
+{
+    Set repo = ##class(%IPM.Repo.Remote.Definition).ServerDefinitionKeyOpen("registry")
+    If $$$AssertTrue($IsObject(repo)) {
+        Do $$$AssertEquals(repo.Enabled,1)
+        Do $$$AssertEquals(repo.URL,"http://registry:52773/registry/")
+        Do $$$AssertEquals(repo.Username,"admin")
+        Do $$$AssertEquals(repo.Password,"SYS")
+    }
+}
+
 }