fix: add %IPM mappings automatically on upgrade

...and a bunch of other/related things:
* Added <Invoke> elements to handle custom setup steps, including mappings (most critically) and language extensions. Also changed version to use -SNAPSHOT as this is what open source automation assumes.
* In IPM.Installer, drive unpacking/installing from EndCompile instead of CreateProjection to avoid lock contention on the class definition itself. (This could be a better solution than asyncSetup in itself - didn't want to go that far in refactoring.)
* Add resources for sandbox container (easier to test installs/upgrades there than via act)
* Refactor other top-level files - setup-registry.sh to scripts
* Some CI tweaks

.github/workflows/main.yml

@@ -142,6 +142,7 @@ jobs:
           | docker exec -i $CONTAINER iris session iris -UUSER
           docker logs $CONTAINER
           docker stop $CONTAINER
+      # NOTE: commented out until 0.7.1 is released.
       # - name: Clean install current version and test new one.
       #   timeout-minutes: 15
       #   run: |
@@ -171,13 +172,13 @@ jobs:
         run: |
           curl http://localhost:52773/registry/packages/-/all | jq
           curl http://localhost:52773/registry/packages/zpm/ | jq
-          ASSET_NAME='zpm-0.7.1-beta.3.xml'
+          ASSET_NAME='zpm-0.7.1-beta.4.xml'
           ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
           wget $ASSET_URL -O /tmp/zpm.xml
           docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}
           CONTAINER=`docker ps -lq`
           docker cp /tmp/zpm.xml $CONTAINER:/home/irisowner/zpm.xml
-          docker cp tests/migration/v0.7-to-v0.9/ $CONTAINER:/tmp/test-package/
+          docker cp tests/migration/v0.7-to-v0.9/. $CONTAINER:/tmp/test-package/
           sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
           /bin/echo -e '' \
             'set sc = ##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm.xml", "ck")\n' \
@@ -196,8 +197,9 @@ jobs:
         timeout-minutes: 15
         run: |
           wget http://localhost:52773/registry/packages/zpm/latest/installer -O /tmp/zpm.xml
-          docker run --network zpm --rm -d -v /tmp/zpm.xml:/home/irisowner/zpm.xml ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}
+          docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }}
           CONTAINER=`docker ps -lq`
+          docker cp /tmp/zpm.xml $CONTAINER:/home/irisowner/zpm.xml
           sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
           /bin/echo -e '' \
             'set sc = ##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm.xml", "ck")\n' \

docker-compose.yml

@@ -25,13 +25,16 @@ services:
       - IRISUSERNAME=admin
       - IRISNAMESPACE=REGISTRY
     volumes:
-      - ./setup-registry.sh:/docker-entrypoint-initdb.d/setup-registry.sh
+      - ./scripts/setup-registry.sh:/docker-entrypoint-initdb.d/setup-registry.sh
   sandbox:
-    image: intersystems/iris-community:preview
+    build: ./tests/sandbox/
     restart: always
     ports: 
       - 52776:52773
     environment:
       - IRISPASSWORD=SYS
       - IRISUSERNAME=admin
       - IRISNAMESPACE=USER
+    volumes:
+      - ./:/home/irisowner/zpm/
+      - ./tests/sandbox/:/home/irisowner/sandbox/

iris.script

@@ -1,5 +1,5 @@
   zn "USER"
-  do $System.OBJ.Load("/home/irisowner/zpm/Installer.cls","ck")
+  do $System.OBJ.Load("/home/irisowner/zpm/preload/cls/IPM/Installer.cls","ck")
   do ##class(IPM.Installer).setup("/home/irisowner/zpm/",3)
   zpm "repo -r -name registry -url https://pm.community.intersystems.com/"
   halt

module.xml

@@ -2,7 +2,7 @@
 <Export generator="Cache" version="25">
 <Document name="ZPM.ZPM"><Module>
   <Name>ZPM</Name>
-  <Version>0.9.0+snapshot</Version>
+  <Version>0.9.0-SNAPSHOT</Version>
   <ExternalName>Package Management System</ExternalName>
   <Description>Provides development tools and infrastructure for defining, building, distributing, and installing modules and applications.</Description>
   <Keywords>Package Manager</Keywords>
@@ -14,9 +14,12 @@
   <Resource Name="%IPM.PKG"/>
   <Resource Name="%IPM.Common.INC"/>
   <Resource Name="%IPM.Formatting.INC"/>
+  <Resource Name="IPM.Installer.CLS" Preload="true" />
   <UnitTest Name="/tests/unit_tests/" Package="Test.PM.Unit" Phase="test"/>
   <UnitTest Name="/tests/integration_tests/" Package="Test.PM.Integration" Phase="verify"/>
   <Resource Name="/doc/README.txt"/>
+  <Invoke Class="IPM.Installer" Method="Map" Phase="Reload" When="Before" />
+  <Invoke Class="%IPM.Main" Method="UpdateLanguageExtensions" />
   <Invoke Class="%IPM.Utils.Migration" Method="RunAll">
     <Arg>${verbose}</Arg>
   </Invoke>

Installer.cls → preload/cls/IPM/Installer.cls

@@ -40,6 +40,18 @@ XData PM [ XMLNamespace = INSTALLER ]
 </Manifest>
 }
 
+XData MapOnly [ XMLNamespace = INSTALLER ]
+{
+<Manifest>
+  <Namespace Name="${CURNS}" Code="${CURNSRoutineDB}" Data="${CURNSGlobalDB}" Create="no" Ensemble="0">
+    <Configuration>
+        <ClassMapping Package="%IPM" From="${CURNSRoutineDB}" />
+        <RoutineMapping Routines="%IPM.*" Type="ALL" From="${CURNSRoutineDB}" /> 
+    </Configuration>
+  </Namespace>
+</Manifest>
+}
+
 /// This is a method generator whose code is generated by XGL.
 ClassMethod setup(ByRef pVars, pLogLevel As %Integer, pInstaller As %Installer.Installer, pLogger As %Installer.AbstractLogger) As %Status [ CodeMode = objectgenerator, Internal ]
 {
@@ -56,6 +68,21 @@ ClassMethod setup(ByRef pVars, pLogLevel As %Integer, pInstaller As %Installer.I
   Quit ##class(%Installer.Manifest).%Generate(%compiledclass, %code, "PM")
 }
 
+ClassMethod Map(ByRef pVars, pLogLevel As %Integer, pInstaller As %Installer.Installer, pLogger As %Installer.AbstractLogger) As %Status [ CodeMode = objectgenerator, Internal ]
+{
+  #; Let our XGL document generate code for this method.
+  Do %code.WriteLine(" Set pVars(""CURNS"")=$namespace")
+  Do %code.WriteLine(" Set prevNS = $namespace")
+  Do %code.WriteLine(" New $Namespace")
+  Do %code.WriteLine(" Set $Namespace = ""%SYS"" ")
+  Do %code.WriteLine(" Kill properties")
+  Do %code.WriteLine(" Set sc = ##class(Config.Namespaces).Get(prevNS, .properties)")
+  Do %code.WriteLine(" Set pVars(""CURNSRoutineDB"")=$Get(properties(""Routines""))")
+  Do %code.WriteLine(" Set pVars(""CURNSGlobalDB"")=$Get(properties(""Globals""))")
+  Do %code.WriteLine(" If ##class(Config.MapPackages).Exists(prevNS,""%IPM"") { Quit $$$OK }")
+  Quit ##class(%Installer.Manifest).%Generate(%compiledclass, %code, "MapOnly")
+}
+
 ClassMethod AsyncSetup(Args...)
 {
   Try {
@@ -88,7 +115,7 @@ ClassMethod ZPMCompile()
   Quit ##class(%IPM.Main).Shell("ZPM compile")
 }
 
-ClassMethod CreateProjection(classname As %String, ByRef parameters As %String, modified As %String, qstruct) As %Status
+ClassMethod EndCompile(qstruct) As %Status
 {
   #; Behave as simple installer
   If (..#FILESCOUNT'>0)||(..#VERSION="") QUIT $$$OK

tests/sandbox/Dockerfile

@@ -0,0 +1,9 @@
+ARG BASE=containers.intersystems.com/intersystems/iris-community:2024.1
+FROM ${BASE}
+
+USER root
+
+RUN apt-get update \
+ && apt-get install -y jq
+
+USER irisowner

tests/sandbox/install-artifact.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+    echo "Enter filename of a previous IPM release asset to install: "
+    read ASSET_NAME
+else
+    ASSET_NAME=$1
+fi
+
+if [ "$ASSET_NAME" = "local" ]; then
+    ASSET_URL="http://registry:52773/registry/packages/zpm/latest/installer"
+else
+    ASSET_URL=`wget -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
+    if [ "$ASSET_URL" = "" ]; then
+        echo "Asset ${ASSET_NAME} not found."
+        exit
+    fi
+fi
+wget $ASSET_URL -O /home/irisowner/zpm.xml
+/bin/echo "
+set sc = ##class(%SYSTEM.OBJ).Load(\"/home/irisowner/zpm.xml\", \"ck\")
+zpm \"list\":1
+zpm \"repo -r -name registry -url http://registry:52773/registry/ -username admin -password SYS\":1
+halt
+" | iris session iris -UUSER