From f62ee826736923e6c3c1333c38443671739a4198 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 26 Apr 2018 16:36:58 +0200
Subject: [PATCH 01/28] core: Shellfe: cleanup

---
 aii-core/src/main/perl/Shellfe.pm     | 587 ++++++++++++++++++--------
 aii-core/src/test/perl/parallel.t     |  34 +-
 aii-core/src/test/perl/shellfe-dhcp.t |  32 +-
 3 files changed, 434 insertions(+), 219 deletions(-)

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index f535b7f9..05229bcb 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -14,9 +14,14 @@ specified in the profile.
 
 Check aii-shellfe for option documentation
 
+=head1 FUNCTIONS
+
+=over
+
 =cut
 
 use CAF::FileWriter;
+use CAF::FileReader;
 use CAF::Lock qw (FORCE_IF_STALE);
 use EDG::WP4::CCM::CacheManager;
 use EDG::WP4::CCM::Fetch::ProfileCache qw($ERROR);
@@ -42,9 +47,8 @@ use 5.10.1;
 use constant MODULEBASE => 'NCM::Component::';
 use constant USEMODULE  => "use " . MODULEBASE;
 use constant PROFILEINFO => 'profiles-info.xml';
+
 use constant NODHCP     => 'nodhcp';
-use constant NONBP      => 'nonbp';
-use constant NOOSINSTALL=> 'noosinstall';
 use constant OSINSTALL  => '/system/aii/osinstall';
 use constant NBP        => '/system/aii/nbp';
 use constant CDBURL     => 'cdburl';
@@ -82,11 +86,23 @@ use constant LIVECDMETHOD    => 'Livecd';
 Readonly our $PROTECTED_COMMANDS => 'remove|configure|(re)?install';
 Readonly our $PROTECTED_OPTION   => 'confirm';
 
+# Keep this list in sync with the options list (to support no<pluginname>)
+# this is also the order in which the plugins run (in iter_plugins)
+# TODO: no dhcp? (but there's a nodhcp option)
+Readonly::Array my @PLUGIN_NAMES => qw(osinstall nbp discovery);
+
+Readonly my $STATE_FILENAME => 'aii-configured';
+
 use parent qw (CAF::Application CAF::Reporter);
 
 our $ec = LC::Exception::Context->new->will_store_errors;
 
-# List of options for this application.
+=item app_options
+
+List of options for this application (extends L<CAF::Application> default list).
+
+=cut
+
 sub app_options
 {
 
@@ -140,13 +156,12 @@ sub app_options
          HELP    => 'File with the nodes to be booted in rescue mode',
          DEFAULT => undef },
 
-       { NAME    => 'include=s',
-         HELP    => 'Directories to add to include path',
+       { NAME    => INCLUDE.'=s',
+         HELP    => 'Directories to add to include path (: delimited list)',
          DEFAULT => '' },
 
        { NAME    => 'status=s',
-         HELP    => 'Report current boot/install status for the node ' .
-         '(can be a regexp)',
+         HELP    => 'Report current boot/install status for the node (can be a regexp)',
          DEFAULT => undef },
 
        { NAME    => 'statuslist=s',
@@ -185,18 +200,19 @@ sub app_options
          HELP    => 'File with the nodes and the actions to perform',
          DEFAULT => undef },
 
-         # aii-* parameters
+       # aii-* parameters
 
-       { NAME    => 'nodiscovery|nodhcp',
-         HELP    => 'Do not update discovery (e.g. dhcp) configuration',
+       # disable plugins
+       { NAME    => 'nodiscovery',
+         HELP    => 'Do not update discovery configuration via discovery plugin',
          DEFAULT => undef },
 
        { NAME    => 'nonbp',
-         HELP    => 'Do not update Network Boot Protocol (e.g. pxe) configuration',
+         HELP    => 'Do not update Network Boot Protocol (e.g. pxe) configuration via nbp plugin',
          DEFAULT => undef },
 
        { NAME    => 'noosinstall',
-         HELP    => 'Do not update OS installer (e.g. kickstart) configuration',
+         HELP    => 'Do not update OS installer (e.g. kickstart) configuration via osinstall plugin',
          DEFAULT => undef },
 
        { NAME    => "$PROTECTED_OPTION=s",
@@ -207,7 +223,12 @@ sub app_options
                     'command invocation.',
          DEFAULT => undef },
 
-         # other common options
+       # DHCP option, not a plugin
+       { NAME    => NODHCP,
+         HELP    => 'Do not update DHCP configuration',
+         DEFAULT => undef },
+
+       # other common options
 
        { NAME    => 'cfgfile=s',
          HELP    => 'configuration file for aii-shellfe defaults',
@@ -250,17 +271,17 @@ sub app_options
          HELP    => "Add process ID to the log messages (disabled by default, unless parallel is > 1)",
          DEFAULT => undef },
 
-         # Options for osinstall plug-ins
+       # Options for osinstall plug-ins
        { NAME    => 'osinstalldir=s',
          HELP    => 'Directory where Kickstart files will be installed',
          DEFAULT => '/osinstall/ks' },
 
-         # Options for DISCOVERY plug-ins
-       { NAME => 'dhcpcfg=s',
-         HELP => 'name of aii configuration file for dhcp',
+       # Options for DISCOVERY plug-ins
+       { NAME    => 'dhcpcfg=s',
+         HELP    => 'name of aii configuration file for dhcp',
          DEFAULT => '/etc/aii/aii-dhcp.conf' },
 
-         # Options for NBP plug-ins
+       # Options for NBP plug-ins
        { NAME    => NBPDIR_PXELINUX.'=s',
          HELP    => 'Directory where files for PXELINUX NBP should be stored',
          DEFAULT => OSINSTALL_DEF_ROOT_PATH . OSINSTALL_DEF_PXELINUX_DIR },
@@ -287,39 +308,36 @@ sub app_options
          HELP    => 'Generic "boot from rescue image" file',
          DEFAULT => 'rescue.cfg' },
 
-     # Options for HTTPS
-       { NAME   => CAFILE.'=s',
-         HELP   => 'Certificate file for the CA' },
+       # Options for HTTPS
+       { NAME    => CAFILE.'=s',
+         HELP    => 'Certificate file for the CA' },
 
-       { NAME   => CADIR.'=s',
-         HELP   => 'Directory where allCA certificates can be found' },
+       { NAME    => CADIR.'=s',
+         HELP    => 'Directory where allCA certificates can be found' },
 
-       { NAME   => KEY.'=s',
-         HELP   => 'Private key for the certificate' },
+       { NAME    => KEY.'=s',
+         HELP    => 'Private key for the certificate' },
 
-       { NAME   => CERT.'=s',
-         HELP   => 'Certificate file to be used' },
+       { NAME    => CERT.'=s',
+         HELP    => 'Certificate file to be used' },
 
        { NAME => "template-path=s",
          HELP => 'store for Template Toolkit files',
-         DEFAULT => '/usr/share/templates/quattor'
-        },
+         DEFAULT => '/usr/share/templates/quattor' },
 
-         # options inherited from CAF
-         #   --help
-         #   --version
-         #   --verbose
-         #   --debug
-         #   --quiet
-
-        );
+       );
 
     return(\@array);
 
 }
 
-# Initializes the application object. Creates the lock and locks the
-# application.
+=item _initialize
+
+Initializes the application object. Creates the lock and locks the
+application.
+
+=cut
+
 sub _initialize
 {
     my $self = shift;
@@ -388,10 +406,15 @@ sub _initialize
 }
 
 
-# Extract CAF::Download::LWP or CCM HTTPS (or other)
-# download related options from aii-shellfe config/options
-# Returns a hashref with options.
-# type can be lwp or ccm
+=item _download_options
+
+Extract C<CAF::Download::LWP> or C<CCM> HTTPS (or other)
+download related options from aii-shellfe config/options
+Returns a hashref with options.
+C<type> can be C<lwp> or C<ccm>.
+
+=cut
+
 sub _download_options
 {
     my ($self, $type) = @_;
@@ -420,9 +443,13 @@ sub _download_options
     return $opts;
 }
 
+=item lock_node
+
+Lock a node being configured, needs to be called in every method that contains
+node operations (ie configure etc)
+
+=cut
 
-# Lock a node being configured, needs to be called in every method that contains
-# node operations (ie configure etc)
 sub lock_node
 {
     my ($self, $node) = @_;
@@ -430,27 +457,39 @@ sub lock_node
     mkdir($self->option("lockdir"));
     my $lockfile = $self->option("lockdir") . "/$node";
     my $lock = CAF::Lock->new ($lockfile, log => $self);
-    if ($lock) {
-        $lock->set_lock (RETRIES, TIMEOUT, FORCE_IF_STALE) or return undef;
+    if ($lock && $lock->set_lock(RETRIES, TIMEOUT, FORCE_IF_STALE)) {
+        $self->debug(3, "aii-shellfe: locked node $node (lockfile $lockfile)");
+        return $lock;
     } else {
-        return undef;
+        $self->debug(3, "aii-shellfe: failed to lock node $node (lockfile $lockfile)");
+        return;
     }
-    $self->debug(3, "aii-shellfe: locked node $node");
-    return $lock;
 }
 
-# Overwrite the report method to allow the KS plug-in to print
-# debugging output. See CAF::Reporter (8) for more information.
+=item report
+
+Overwrite the report method to allow the KS plug-in to print
+debugging output. See L<CAF::Reporter> for more information.
+
+=cut
+
 sub report
 {
     my $self = shift;
-    my $st = join ('', @_);
-    print STDOUT "$st\n" unless $SUPER::_REP_SETUP->{QUIET};
+    my $msg = join ('', @_);
+    print STDOUT "$msg\n" unless $SUPER::_REP_SETUP->{QUIET};
     $self->log (@_);
     return SUCCESS;
 }
 
-sub plugin_handler {
+=item plugin_handler
+
+Handler for exceptions during plugin run
+
+=cut
+
+sub plugin_handler
+{
     my ($self, $plugin, $ec, $e) = @_;
     $self->error("$plugin: $e");
     $self->{status} = PARTERR_ST;
@@ -458,25 +497,36 @@ sub plugin_handler {
     return;
 }
 
-# Runs $method on the plug-in given at $path for $node. Arguments:
-# 1: the node state (value of hash returned by fetch_profiles)
-# 2: the PAN path of the plug-in to be run. If the path does not
-#     exist, nothing will be done.
-# 3: the method to be run.
-# 4: optional modulename: when provided, use module with that name
-#     (PAN path is ignored when determining module(s) to use).
-#     when none is provided, all keys of the PAN path will be used as modules.
+=item run_plugin
+
+Runs C<method> on the plug-in given at C<path> for node state C<st>
+(value of hash returned by fetch_profiles).
+
+C<path> is the PAN path of the plug-in to be run. If the path does not
+exist, nothing will be done.
+
+Optional C<modulename> to select the name of the module.
+When provided, use module with that name (PAN path is ignored when determining module(s) to use).
+When none is provided, all keys of the PAN path will be used as modules.
+
+=cut
+
+# all plugins should return success on success
+
 sub run_plugin
 {
     my ($self, $st, $path, $method, $only_modulename) = @_;
 
-    return unless $st->{configuration}->elementExists ($path);
+    my $name = $st->{name};
+    my $tree = $st->{configuration}->getTree($path);
+    if (!$tree) {
+        $self->verbose("No configuration for plugin path $path for $name. Skipping");
+        return;
+    }
 
     # This is here because CacheManager and Fetch objects may have
     # problems when they get out of scope.
-    my %rm = $st->{configuration}->getElement ($path)->getHash;
-
-    my @modules = $only_modulename ? ($only_modulename) : sort keys %rm;
+    my @modules = $only_modulename ? ($only_modulename) : sort keys %$tree;
 
     # Iterate over module names, handling each
     foreach my $modulename (@modules) {
@@ -487,7 +537,10 @@ sub run_plugin
         }
 
         local $@;
-        if (!exists $self->{plugins}->{$modulename}) {
+
+        my $plug = $self->{plugins}->{$modulename};
+
+        if (!defined $plug) {
             $self->debug (4, "Loading plugin module $modulename");
             eval (USEMODULE . $modulename);
             if ($@) {
@@ -495,138 +548,174 @@ sub run_plugin
                 $self->{status} = PARTERR_ST;
                 next;
             }
+
             $self->debug (4, "Instantiating $modulename");
             my $class = MODULEBASE.$modulename;
 
             # Plugins as derived from NCM::Component, so they need a name argument
-            my $module = eval { $class->new($modulename) };
+            $plug = eval { $class->new($modulename) };
             if ($@) {
                 $self->error ("Couldn't call 'new' on plugin module $modulename: $@");
                 $self->{status} = PARTERR_ST;
-                next;
+                return;
             }
-            $self->{plugins}->{$modulename} = $module;
+
+            $self->{plugins}->{$modulename} = $plug;
         }
 
-        my $plug = $self->{plugins}->{$modulename};
         if ($plug->can($method)) {
-            $self->debug (4, "Running plugin module $modulename -> $method");
+            $self->debug (4, "Running plugin module $modulename -> $method for $name");
             $aii_shellfev2::__EC__ = LC::Exception::Context->new;
             $aii_shellfev2::__EC__->error_handler(sub {
                 $self->plugin_handler($modulename, @_);
             });
 
-            if (!eval { $plug->$method ($st->{configuration}) }) {
-                $self->error ("Failed to execute plugin module's $modulename $method method");
-                $self->{status} = PARTERR_ST;
-            }
+            # The plugin method has to return success
+            my $res = eval { $plug->$method ($st->{configuration}) };
             if ($@) {
-                $self->error ("Errors running plugin module $modulename $method method: $@");
+                $self->error ("Errors running plugin module $modulename $method method for $name: $@");
+                $self->{status} = PARTERR_ST;
+            } elsif (! $res) {
+                $self->error ("Failed to execute plugin module $modulename $method method for $name");
                 $self->{status} = PARTERR_ST;
             }
         } else {
-            $self->debug(4, "no method $method available for plugin module $modulename");
+            # TODO: should be warn or error? it's configured, but the code doesn't allow it do anything
+            $self->info("no method $method available for plugin module $modulename for $name");
         }
     }
+
+    return;
 }
 
-# Call AII::DHCP with the configuration object received as argument. It
-# uses the MAC of the first card marked with "boot"=true.
-# dhcpmgr is the instance of AII:DHCP that collects the nodes to configure or remove
+=item dhcp
+
+Runs aii-dhcp on the configuration object received as argument. It
+uses the MAC of the first card marked with C<<"boot"=true>>.
+
+=cut
+
 sub dhcp
 {
-    my ($self, $node, $st, $cmd, $dhcpmgr) = @_;
+    my ($self, $st, $cmd, $dhcpmgr) = @_;
 
-    return unless $st->{configuration}->elementExists (DHCPPATH);
+    my $name = $st->{name};
+    my $tree = $st->{configuration}->getTree(DHCPPATH);
+    if (! $tree) {
+        $self->verbose("No configuration for DHCP path ".DHCPPATH." for $name. Skipping");
+        return;
+    }
 
     my $mac;
-    my $cards = $st->{configuration}->getElement (HWCARDS)->getTree;
-    foreach my $cfg (sort(values (%$cards))) {
-       if ($cfg->{boot}) {
-           $cfg->{hwaddr} =~ m{^((?:[0-9a-f]{2}[-:])+(?:[0-9a-f]{2}))$}i;
-           $mac = $1;
-           last;
-       }
+    my $cards = $st->{configuration}->getTree(HWCARDS);
+    foreach my $cfg (sort values (%$cards)) {
+        if ($cfg->{boot}) {
+            if ($cfg->{hwaddr} =~ m{^((?:[0-9a-f]{2}[-:])+(?:[0-9a-f]{2}))$}i) {
+                $mac = $1;
+                $self->verbose("Using macaddress from (first) boot nic");
+                last;
+            };
+        }
     }
 
     my $ec;
     if ($cmd eq CONFIGURE) {
-        my $opts = $st->{configuration}->getElement (DHCPOPTION)->getTree;
-        $self->debug (4, "Going to add dhcp entry of $node to configure");
-        $ec = $dhcpmgr->new_configure_entry($node, $mac, $opts->{tftpserver} // '', $opts->{addoptions} // ());
+        my $opts = $st->{configuration}->getTree(DHCPOPTION);
+        $self->debug (4, "Going to add dhcp entry of $name to configure");
+        $ec = $dhcpmgr->new_configure_entry($name, $mac, $opts->{tftpserver} // '', $opts->{addoptions} // ());
     } elsif ($cmd eq REMOVEMETHOD) {
         if ($st->{reinstall}) {
-            $self->debug(3, "No dhcp removal with reinstall set for $node");
+            $self->debug(3, "No dhcp removal with reinstall set for $name");
         } else {
-            $self->debug (4, "Going to add dhcp entry of $node to remove");
-            $ec = $dhcpmgr->new_remove_entry($node);
+            $self->debug (4, "Going to add dhcp entry of $name to remove");
+            $ec = $dhcpmgr->new_remove_entry($name);
         }
     } else {
-        $self->error("on node $node: dhcp should only run for configure and remove methods, not $cmd");
+        $self->error("$name: dhcp should only run for configure and remove methods, not $cmd");
         $ec = 1;
     }
     if ($ec) {
-        $self->error("Error running method $cmd on node $node");
+        $self->error("Error running method $cmd on $name");
     }
 }
 
+=item iter_plugins
+
+Given node state and optional hook, iterate over all plugins in PLUGIN_NAMES.
+
+=cut
 
 sub iter_plugins
 {
     my ($self, $st, $hook) = @_;
-    foreach my $plug (qw(osinstall nbp discovery)) {
-        my $path = "/system/aii/$plug";
-        if (!$self->option("no$plug")) {
+    foreach my $pluginname (@PLUGIN_NAMES) {
+        my $path = "/system/aii/$pluginname";
+        if ($self->option("no$pluginname")) {
+            $self->verbose("no$pluginname option set, skipping $pluginname plugin");
+        } else {
             $self->run_plugin($st, $path, $hook);
         }
     }
 }
 
 
-# Returns an array with the list of nodes specified in the file given
-# as an argument. Arguments:
-#
-# $_[1]: file name containing the list of nodes. Each element of the
-# list can be a regular expression!
-# $_[2]: whether or not the fully qualified domain name should be used
-# in the profile name.
+=item filenodelist
+
+Returns an array with the list of nodes specified in the filename given
+as an argument. Each element of the list can be a regular expression.
+
+The second argument is a boolean whether or not the
+fully qualified domain name should be used in the profile name.
+
+=cut
+
 sub filenodelist
 {
-    my ($self, $rx, $fqdn) = @_;
+    my ($self, $filename, $fqdn) = @_;
 
     my @nl;
 
-    open (FH, "<$rx") or throw_error ("Couldn't open file: $rx");
+    my $fh = CAF::FileReader->new($filename, log => $self);
 
-    while (my $l = <FH>) {
+    while (my $l = <$fh>) {
         next if $l =~ m/^#/;
         chomp ($l);
-        $self->debug (3, "Evaluating regexp $l");
+        $self->debug(3, "Evaluating regexp $l");
         push (@nl, $self->nodelist ($l, $fqdn));
     }
-    close (FH);
-    $self->debug (1, "Node list: " . join ("\t", @nl));
+    $self->debug (1, "Node list from $filename: " . join (", ", @nl));
     return @nl;
 }
 
-# Returns the list of profiles on the CDB server that match a given
-# regular expression.
-#
-# Arguments:
-# $_[1]: the regular expression.
-# $_[2]: whether or not to use fully qualified domain names in the
-# profiles names.
+=item nodelist
+
+Returns the list of profiles that match a given regular expression.
+
+The second argument is a boolean whether or not the
+fully qualified domain name should be used in the profile name.
+
+The original list of all known profiles is determined once
+based on the C<cdburl> option.
+
+=cut
+
 sub nodelist
 {
-    my ($self, $rx, $fqdn) = @_;
+    my ($self, $pattern, $fqdn) = @_;
+
+    my $orig_pattern = $pattern;
     # allow the nodename to be specified as either simple nodename, or
-    # as filename (i.e. .xml). However, to make sure our regexes make
+    # as filename (i.e. .xml or .json.gz).
+    # However, to make sure our regexes make
     # sense, we normalize to forget about the .xml for now.
+    # TODO: we are actually normalizing a regex pattern, which is insane
+
     my $extension = '\.(?:xml|json)(?:\.gz)?$';
-    $rx =~ s{$extension}{};
-    my $prefix = $self->option (PREFIX) || '';
+    $pattern =~ s{$extension}{};
+    my $prefix = $self->option(PREFIX) || '';
 
     if (!$profiles_info) {
+        # Populate the module variable profiles_info with all known profile names
         if ($self->option (CDBURL) =~ m{^dir://(.*)$} ) {
             my $dir = $1;
             $self->debug (4, "Creating profiles-info from local directory $dir");
@@ -653,24 +742,36 @@ sub nodelist
         };
     }
 
-    $rx =~ m{^([^.]*)(.*)};
-    $rx = $1;
-    $rx .= "($2)" if $fqdn;
+    if ($pattern =~ m{^([^.]*)(.*)}) {
+        $pattern = $1;
+        $pattern .= "($2)" if $fqdn;
+    };
+
     my @nl;
-    foreach (@{$profiles_info->{profile}}) {
-        if ($_->{content} =~ m/$prefix($rx)\.(?:xml|json)\b/) {
+    foreach my $profile (@{$profiles_info->{profile}}) {
+        if ($profile->{content} =~ m/$prefix($pattern)$extension\b/) {
             my $host = $1;
             $self->debug (4, "Added $host to the list");
             push (@nl, $host);
         }
     }
 
-    $self->error ("No node matches $rx") unless (@nl);
+    $self->error ("No node matches $pattern (original $orig_pattern)") unless (@nl);
     return @nl;
 }
 
-sub cachedir {
+=item cachedir
+
+Generate the name of the node cachedir from the node name and the C<cachedir> option.
+
+If the C<use_fqdn> opion is used, and additional subdirectory is used (with domainname as value).
+
+=cut
+
+sub cachedir
+{
     my ($self, $node) = @_;
+
     my $basedir = $self->option("cachedir");
     my $cachedir = $basedir;
     if ($self->option('use_fqdn') and $node =~ m{\.(.*)}) {
@@ -684,13 +785,19 @@ sub cachedir {
     return $cachedir;
 }
 
-# Returns a hash with the node names given as arguments as keys and
-# the pair { fetch, cachemanager } objects associated to their
-# profiles as values.
+=item fetch_profiles
+
+Returns a hash with the node names given as arguments as keys and
+a hashref with the C<name> and C<fetch>, C<cachemanager> and C<configuration> instances
+associated to their profiles as values.
+
+(This hashref is sometimes referred to as the node state in this code).
+
+=cut
+
 sub fetch_profiles
 {
     my ($self, @nl) = @_;
-    my %h;
 
     my $cdb = $self->option (CDBURL);
     my $prefix = $self->option (PREFIX) || '';
@@ -699,9 +806,9 @@ sub fetch_profiles
     if ($suffix =~ m{^([-\w\.]*)$}) {
         $suffix = $1;
     } else {
-        $self->error ("Invalid suffix for profiles. Leaving");
+        $self->error ("Invalid suffix $suffix for profiles");
         $self->{status} = PARTERR_ST;
-        return ();
+        return;
     }
 
     if ($cdb =~ m{([\w\-\.+]+://[+\w\.\-%?=/:]+)}) {
@@ -709,9 +816,9 @@ sub fetch_profiles
         # All profiles from dir:// can be accessed as file://
         $cdb =~ s{^dir://}{file://};
     } else {
-        $self->error ("Invalid base URL. Leaving");
+        $self->error ("Invalid base URL $cdb");
         $self->{status} = PARTERR_ST;
-        return ();
+        return;
     }
 
     # Read the config of the current host
@@ -723,8 +830,11 @@ sub fetch_profiles
     # for each foreign profile
     EDG::WP4::CCM::CCfg::resetCfg();
 
+    my %node_states;
     foreach my $node (@nl) {
-        next if exists $h{$node};
+        # ignore duplicate entries
+        next if exists $node_states{$node};
+
         my $ccmdir = $self->cachedir($node);
         my $url = "$cdb/$prefix$node.$suffix";
         $self->debug (1, "Fetching profile: $url");
@@ -737,7 +847,7 @@ sub fetch_profiles
 
         my $cfg_fh = CAF::FileWriter->new($config, log => $self);
         my $err = $ec->error();
-        if(defined($err)) {
+        if (defined($err)) {
             $self->error("failed to create config file $config: ".$err->reason());
             next;
         } else {
@@ -757,7 +867,12 @@ sub fetch_profiles
         };
 
         # we use CDB_File, since it's the fastest
-        my  $fh = EDG::WP4::CCM::Fetch->new ({PROFILE_URL => $url, FOREIGN => 1, CONFIG => $config, DBFORMAT => 'CDB_File'});
+        my  $fh = EDG::WP4::CCM::Fetch->new ({
+            PROFILE_URL => $url,
+            FOREIGN => 1,
+            CONFIG => $config,
+            DBFORMAT => 'CDB_File', # CDB_File is the fastest
+            });
         unless ($fh) {
             $self->error ("Error creating Fetch object for $url");
             $self->{status} = PARTERR_ST;
@@ -777,22 +892,22 @@ sub fetch_profiles
             next;
         }
 
-        my $cm = EDG::WP4::CCM::CacheManager->new ($fh->{CACHE_ROOT}, $config);
+        my $cm = EDG::WP4::CCM::CacheManager->new($fh->{CACHE_ROOT}, $config);
         if ($cm) {
-            my $cfg = $cm->getLockedConfiguration (0);
-            $h{$node} = {
+            my $cfg = $cm->getLockedConfiguration(0);
+            $node_states{$node} = {
+                name         => $node,
                 fetch        => $fh,
                 cachemanager => $cm,
                 configuration=> $cfg,
             };
         } else {
-            $self->error ("Failed to create CacheManager ",
-                          "object for node $node");
+            $self->error ("Failed to create CacheManager object for node $node. Skipping node.");
             $self->{status} = PARTERR_ST;
         }
         $self->debug (1, "Inserted structure for $node on fetching structure");
     }
-    return %h;
+    return %node_states;
 }
 
 # Initiate the Parallel:ForkManager with requested threads if option is given
@@ -821,7 +936,8 @@ sub init_pm
 }
 
 # Run the cmd on the list of nodes
-sub run_cmd {
+sub run_cmd
+{
     my ($self, $cmd, %node_states) = @_;
     my $method = "_$cmd";
     my %responses;
@@ -870,16 +986,26 @@ foreach my $cmd (COMMANDS) {
 use strict 'refs';
 
 
-# Runs the Install method of the NBP plugins of the nodes given as
-# arguments.
+# All the _ methods below should return 0 or undef as success.
+
+=item _install
+
+Runs the Install method of the NBP plugins of the node.
+
+=cut
+
 sub _install
 {
     my ($self, $node, $st) = @_;
     $self->run_plugin ($st, NBP, INSTALLMETHOD);
 }
 
-# Runs the Status method of the NBP plugins of the nodes given as
-# arguments.
+=item _status
+
+Runs the Status method of the NBP plugins of the node.
+
+=cut
+
 sub _status
 {
     my ($self, $node, $st) = @_;
@@ -888,32 +1014,51 @@ sub _status
     $self->run_plugin ($st, NBP, STATUSMETHOD);
 }
 
-# Runs the Boot method of the NBP plugins of the nodes given as
-# arguments.
+=item _boot
+
+Runs the Boot method of the NBP plugins of the node.
+
+=cut
+
 sub _boot
 {
     my ($self, $node, $st) = @_;
     $self->run_plugin ($st, NBP, BOOTMETHOD);
 }
 
-# Runs the Firmware method of the NBP plugins of the nodes given as
-# arguments.
+=item _firmware
+
+Runs the Firmware method of the NBP plugins of the node.
+
+=cut
+
 sub _firmware
 {
     my ($self, $node, $st) = @_;
     $self->run_plugin ($st, NBP, FIRMWAREMETHOD);
 }
 
-# Runs the Livecd method of the NBP plugins of the nodes given as
-# arguments
+=item _livecd
+
+Runs the Livecd method of the NBP plugins of the node.
+
+=cut
+
 sub _livecd
 {
     my ($self, $node, $st) = @_;
     $self->run_plugin($st, NBP, LIVECDMETHOD);
 }
 
-# Runs the Remove method of the NBP plugins of the nodes given as
-# arguments.
+=item _remove
+
+Runs the Unconfigure method of all plugins of the node.
+
+If the node is not being reinstalled, also runs dhcp
+and removes the cache dir (unless noaction is set).
+
+=cut
+
 sub _remove
 {
     my ($self, $node, $st) = @_;
@@ -927,8 +1072,12 @@ sub _remove
     };
 }
 
-# Runs the Rescue method of the NBP plugins of the nodes given as
-# arguments.
+=item _rescue
+
+Runs the Rescue method of the NBP plugins of the node.
+
+=cut
+
 sub _rescue
 {
     my ($self, $node, $st) = @_;
@@ -936,47 +1085,88 @@ sub _rescue
     $self->run_plugin ($st, NBP, RESCUEMETHOD);
 }
 
-# Configures DISCOVERY, OSINSTALL and NBP on the nodes received as
-# arguments.
+=item _configure
+
+Runs the Configure method of all plugins and dhcp of the node.
+
+=cut
+
 sub _configure
 {
     my ($self, $node, $st) = @_;
 
     my $when = time();
 
-    $self->iter_plugins($st, CONFIGURE);
+    my $res = $self->iter_plugins($st, CONFIGURE);
     $self->set_cache_time($node, $when) unless $self->option('noaction');
+    return $res;
 }
 
+=item _metaconfig
+
+Runs the aii_command method of the metaconfig component of the node.
+
+=cut
+
 sub _metaconfig
 {
     my ($self, $node, $st) = @_;
     $self->run_plugin($st, '/software/components/metaconfig', 'aii_command', 'metaconfig');
 }
 
-sub get_cache_time {
+=item get_cache_time
+
+Return the mtime of the C<node> AII statefile.
+
+=cut
+
+sub get_cache_time
+{
     my ($self, $node) = @_;
 
     my $cachedir = $self->cachedir($node);
-    return (stat("$cachedir/aii-configured"))[9] || 0;
+    return (stat("$cachedir/$STATE_FILENAME"))[9] || 0;
 }
 
+=item set_cache_time
+
+Set the mtime of the C<node> AII statefile to C<when>.
+
+=cut
+
 sub set_cache_time {
     my ($self, $node, $when) = @_;
+
     my $cachedir = $self->cachedir($node);
-    if (!open(TOUCH, ">$cachedir/aii-configured")) {
-        $self->error("aii-shellfe: failed to update state for $node: $!");
-    }
-    close(TOUCH);
+    my $filename = "$cachedir/$STATE_FILENAME";
+
+    my $fh = CAF::FileWriter->new($filename, mtime => $when, log => $self);
+    print $fh '';
+    $fh->close();
 }
 
-sub remove_cache_node {
+=item remove_cache_node
+
+Remove the C<node> cachedir.
+(Does not check the noaction option)
+
+=cut
+
+sub remove_cache_node
+{
     my ($self, $node) = @_;
     my $cachedir = $self->cachedir($node);
     rmtree($cachedir);
 }
 
-# If a host is protected, check the protectid is correct, else don't include to process further
+=item check_protected
+
+Given a hash with node states (as returned by C<fetch_profiles>),
+remove all protected hosts whose C</system/aii/protected> value does
+not match the confirm option.
+
+=cut
+
 sub check_protected {
     my ($self, %hash) = @_;
     my @to_delete;
@@ -1004,19 +1194,28 @@ sub check_protected {
     return %hash;
 }
 
+=item change_dhcp
+
+Make dhcp changes for nodes
+
+=cut
+
 sub change_dhcp
 {
     my ($self, $method, %nodes) = @_;
+
     $self->debug(5, "logfile:", $self->option('logfile'), " dhcpcfg:", $self->option('dhcpcfg'));
-    my $dhcpmgr = AII::DHCP->new('script',
+    my $dhcpmgr = AII::DHCP->new(
+        'script',
         "--logfile=".$self->option('logfile'),
         "--cfgfile=".$self->option('dhcpcfg'),
-        log => $self);
+        log => $self,
+        );
     foreach my $node (sort keys %nodes) {
         my $st = $nodes{$node};
         $self->debug(3, "Checking dhcp config on node $node for method $method.");
         if ($st->{configuration}->elementExists(DHCPPATH)) {
-            $self->dhcp($node, $st, $method, $dhcpmgr);
+            $self->dhcp($st, $method, $dhcpmgr);
         }
     }
     $dhcpmgr->configure_dhcp();
@@ -1024,8 +1223,12 @@ sub change_dhcp
     return 1;
 }
 
+=item cmds
+
+Runs all the commands
+
+=cut
 
-# Runs all the commands
 sub cmds
 {
     my $self = shift;
@@ -1121,15 +1324,29 @@ sub cmds
 
 }
 
-sub finish {
+=item finish
+
+Run the finish method of all plugins
+
+=cut
+
+sub finish
+{
     my ($self) = @_;
     $self->debug(5, "closing down");
-    foreach my $plugin (keys %{$self->{plugins}}) {
-        if ($self->{plugins}->{$plugin}->can("finish")) {
-            $self->debug(5, "invoking finish for $plugin");
-            $self->{plugins}->{$plugin}->finish();
+    foreach my $pluginname (@PLUGIN_NAMES) {
+        my $plugin = $self->{plugins}->{$pluginname};
+        if ($plugin && $plugin->can('finish')) {
+            $self->debug(5, "invoking finish for $pluginname");
+            $plugin->finish();
         }
     }
 }
 
+=pod
+
+=back
+
+=cut
+
 1;
diff --git a/aii-core/src/test/perl/parallel.t b/aii-core/src/test/perl/parallel.t
index e0364a2e..8868ffff 100644
--- a/aii-core/src/test/perl/parallel.t
+++ b/aii-core/src/test/perl/parallel.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use CAF::Object;
-use Test::Deep;
 use Test::More;
 use Test::Quattor qw(basic);
 use Test::MockModule;
@@ -18,12 +17,8 @@ $CAF::Object::NoAction = 1;
 my $caflock = Test::MockModule->new('CAF::Lock');
 my $cfg_basic = get_config_for_profile('basic');
 my $config_basic = { configuration => $cfg_basic };
-my %h = (
-    'test01.cluster' => $config_basic,
-    'test02.cluster' => $config_basic,
-    'test03.cluster' => $config_basic,
-    'test04.cluster' => $config_basic,
-);
+my %h = (map {("test$_.cluster", {configuration => $cfg_basic, name => "test$_.cluster"})} qw(01 02 03 04));
+
 my $defres = {};
 foreach my $host (keys %h) {
 	$defres->{$host} = {
@@ -34,9 +29,11 @@ foreach my $host (keys %h) {
 };
 
 $caflock->mock('set_lock', 1 );
-my @opts = qw(script --logfile=target/test/parallel.log --cfgfile=src/test/resources/parallel.cfg);
-
-
+my @opts = qw(script
+    --logfile target/test/parallel.log
+    --cfgfile src/test/resources/parallel.cfg
+    --debug 5
+);
 
 my $mod = AII::Shellfe->new(@opts);
 
@@ -49,42 +46,43 @@ foreach my $host (keys %h) {
 };
 
 my $ok = $mod->remove(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+is_deeply( $ok, $defres, 'correct result' ) ;
 
 $mod = AII::Shellfe->new(@opts, "--parallel", 2 );
 ($pm, %responses) = $mod->init_pm('test');
-ok($pm, 'parallel fork manager initiated');
+ok($pm, 'parallel fork manager initiated p=2');
 
 foreach my $host (keys %h) {
 	$defres->{$host}->{mode} = 1;
 };
 $ok = $mod->remove(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+is_deeply($ok, $defres, 'correct remove resul p=2t' ) ;
 
 foreach my $host (keys %h) {
 	$defres->{$host}->{method} = '_status';
 };
 $ok = $mod->status(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+is_deeply($ok, $defres, 'correct status result p=2' ) ;
 
 
 $mod = AII::Shellfe->new(@opts, "--parallel", 4 );
 ($pm, %responses) = $mod->init_pm('test');
-ok($pm, 'parallel fork manager initiated');
+ok($pm, 'parallel fork manager initiated p=4');
 
 $ok = $mod->status(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+is_deeply( $ok, $defres, 'correct result p=4' ) ;
 
 foreach my $host (keys %h) {
 	$defres->{$host}->{method} = '_configure';
 };
 $ok = $mod->configure(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+diag " configure p=4 ok ", explain $ok, explain $defres;
+is_deeply( $ok, $defres, 'correct configure result p=4' ) ;
 
 foreach my $host (keys %h) {
 	$defres->{$host}->{method} = '_install';
 };
 $ok = $mod->install(%h);
-cmp_deeply( $ok, $defres, 'correct result' ) ;
+is_deeply( $ok, $defres, 'correct install result p=4' ) ;
 
 done_testing();
diff --git a/aii-core/src/test/perl/shellfe-dhcp.t b/aii-core/src/test/perl/shellfe-dhcp.t
index 2c1c23aa..2d27027a 100644
--- a/aii-core/src/test/perl/shellfe-dhcp.t
+++ b/aii-core/src/test/perl/shellfe-dhcp.t
@@ -41,24 +41,24 @@ my $cfg_3 = { configuration => get_config_for_profile('shellfe-dhcp-3') };
 my $cfg_4 = { configuration => get_config_for_profile('shellfe-dhcp-4') };
 my $cfg_b = { configuration => get_config_for_profile('shellfe-dhcp-b') };
 
-my %configure = ( 
-    'host1.example.com' => $cfg_1,
-    'host2.example.com' => $cfg_2,
-    'host3.example1.com' => $cfg_3,
-    'host4.example.com' => $cfg_4,
-);
-
-my %remove = (
-    'host1.example.com' => $cfg_1,
-    'host5.example.com' => $cfg_b,
-    'host6.example1.com' => $cfg_b,
-);
-
-
-
+sub mk_node_state
+{
+  return map {("host$_.example".(($_ == 3 || $_ == 6) ? 1 : '').".com" => {
+                                       name => "host$_.example".(($_ == 3 || $_ == 6) ? 1 : '').".com",
+                                       configuration => get_config_for_profile("shellfe-dhcp-".($_ > 4 ? 'b' : $_ ))
+                                      }
+             )} @_;
+}
 
+my %configure = mk_node_state(1, 2, 3, 4);
+my %remove = mk_node_state(1, 5, 6);
 
-my @opts = qw(script --logfile=target/test/dhcp.log  --cfgfile=src/test/resources/shellfe.cfg --dhcpcfg=src/test/resources/dhcp.cfg);
+my @opts = qw(script
+    --logfile target/test/shellfe-dhcp.log
+    --cfgfile src/test/resources/shellfe.cfg
+    --dhcpcfg src/test/resources/dhcp.cfg
+    --debug 5
+);
 
 
 my $dhcpd = <<EOF;

From 91c9c62e1f0b55acc496e94187b6873925c10338 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 21 Sep 2016 14:02:57 +0200
Subject: [PATCH 02/28] Shellfe: set active config before running the plugin
 method

---
 aii-core/src/main/perl/Shellfe.pm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index 05229bcb..1244fbef 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -570,6 +570,11 @@ sub run_plugin
                 $self->plugin_handler($modulename, @_);
             });
 
+            # Set active config
+            if ($plug->can('set_active_config')) {
+                $plug->set_active_config($st->{configuration});
+            }
+
             # The plugin method has to return success
             my $res = eval { $plug->$method ($st->{configuration}) };
             if ($@) {

From 963ac1f900811fd4d75b8597a43ba76e964328ad Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 22 Sep 2016 08:24:21 +0200
Subject: [PATCH 03/28] ks: ksuserhooks: set active config before running the
 hook method

---
 aii-ks/src/main/perl/ks.pm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index cf60ee0a..690f502d 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -390,6 +390,10 @@ sub ksuserhooks
             throw_error ("Couldn't instantiate object of hook class $hook->{module} ($modulename): $@");
         } else {
             if ($hook_inst->can($method)) {
+                if ($hook_inst->can('set_active_config')) {
+                    $hook_inst->set_active_config($config);
+                }
+
                 $this_app->debug (5, "Running hook $hook->{module} method $method ($modulename->$method)");
                 $hook_inst->$method ($config, "$path/$idx");
             } else {

From f1f22a1f815ec9ade51578a7939e5c220bcc60c5 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 7 Aug 2020 11:48:03 +0200
Subject: [PATCH 04/28] aii-ks: fix some more regex tests after new
 nmc-lib-blockdevices

---
 .../resources/regexps/pre_noblock_functions   | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/aii-ks/src/test/resources/regexps/pre_noblock_functions b/aii-ks/src/test/resources/regexps/pre_noblock_functions
index 4104776d..eb075646 100644
--- a/aii-ks/src/test/resources/regexps/pre_noblock_functions
+++ b/aii-ks/src/test/resources/regexps/pre_noblock_functions
@@ -4,6 +4,39 @@ Test the functions in the pre section
 ^echo 'Begin of pre section'$
 ^set -x$
 ^$
+^wipe_metadata \(\) \{$
+^\s{4}local path clear SIZE ENDSEEK ENDSEEK_OFFSET$
+^\s{4}path="\$1"$
+^$
+^\s{4}# default to 1$
+^\s{4}clearmb="\$\{2:-1\}"$
+^$
+^\s{4}# wipe at least 4 MiB at begin and end$
+^\s{4}ENDSEEK_OFFSET=4$
+^\s{4}if \[ "\$clearmb" -gt \$ENDSEEK_OFFSET \]; then$
+^\s{8}ENDSEEK_OFFSET=\$clearmb$
+^\s{4}fi$
+^\s{4}\# try to get the size with fdisk$
+^\s{4}SIZE=`disksize_MiB "\$path"`$
+^$
+^\s{4}\# if empty, assume we failed and try with parted$
+^\s{4}if \[ \$SIZE -eq 0 \]; then$
+^\s{8}\# the SIZE has not been determined,$
+^\s{8}\# set it equal to ENDSEEK_OFFSET, the entire disk gets wiped.$
+^\s{8}SIZE=\$ENDSEEK_OFFSET$
+^\s{8}echo "\[WARN\] Could not determine the size of device \$path with both fdisk and parted. Wiping whole drive instead"$
+^\s{4}fi$
+^$
+^\s{4}let ENDSEEK=\$SIZE-\$ENDSEEK_OFFSET$
+^\s{4}if \[ \$ENDSEEK -lt 0 \]; then$
+^\s{8}ENDSEEK=0$
+^\s{4}fi$
+^\s{4}echo "\[INFO\] wipe path \$path with SIZE \$SIZE and ENDSEEK \$ENDSEEK"$
+^\s{4}\# dd with 1 MiB blocksize \(unit used by disksize_MiB and faster then e.g. bs=512\)$
+^\s{4}dd if=/dev/zero of="\$path" bs=1048576 count=\$ENDSEEK_OFFSET$
+^\s{4}dd if=/dev/zero of="\$path" bs=1048576 seek=\$ENDSEEK$
+^\s{4}sync$
+^\}$
 ^$
 ^disksize_MiB \(\) \{$
 ^\s{4}local path BYTES MB RET$
@@ -46,4 +79,3 @@ Test the functions in the pre section
 ^\s{4}echo "\[\$msg\] Found path \$path size \$SIZE min \$min max \$max"$
 ^\s{4}return \$RET$
 ^\}$
-^$

From e685e9ccc6ef0133ce362a695a2536aec93136f0 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Tue, 5 Nov 2019 08:27:22 +0100
Subject: [PATCH 05/28] aii-ks: factor out proxy url modification

---
 aii-ks/src/main/perl/ks.pm | 47 ++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 690f502d..5812cef9 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -415,16 +415,8 @@ sub kscommands
     my @packages = @{$tree->{packages}};
     push(@packages, 'bind-utils'); # required for nslookup usage in ks-post-install
 
-    my $installtype = $tree->{installtype};
-    if ($installtype =~ /http/) {
-        my ($proxyhost, $proxyport, $proxytype) = proxy($config);
-        if ($proxyhost && $proxytype eq "reverse") {
-            if ($proxyport) {
-                $proxyhost .= ":$proxyport";
-            }
-            $installtype =~ s{(https?)://([^/]*)/}{$1://$proxyhost/};
-        }
-    }
+    my $proxy_config = [proxy($config)];
+    my $installtype = proxy_url($proxy_config, $tree->{installtype});
 
     my $ntp_servers = '';
     if ($tree->{ntpservers} && $version >= ANACONDA_VERSION_EL_7_0) {
@@ -442,19 +434,12 @@ timezone --utc $tree->{timezone}$ntp_servers
 rootpw --iscrypted $tree->{rootpw}
 EOF
 
-    if ($tree->{repo}) {
-        foreach my $url (@{$tree->{repo}}) {
-            if ($url =~ /http/) {
-                my ($proxyhost, $proxyport, $proxytype) = proxy($config);
-                if ($proxyhost && $proxytype eq "reverse") {
-                    if ($proxyport) {
-                        $proxyhost .= ":$proxyport";
-                    }
-                    $url =~ s{(https?)://([^/]*)/}{$1://$proxyhost/};
-                }
-            }
-            print "repo $url\n";
+    foreach my $url (@{$tree->{repo} || []}) {
+        if ($url =~ m/^@(.+)$/) {
+        } else {
+            $url = proxy_url($proxy_config, $url);
         }
+        print "repo $url\n";
     }
 
     if ($tree->{cmdline}) {
@@ -1031,6 +1016,24 @@ sub proxy
     return ($proxyhost, $proxyport, $proxytype);
 }
 
+# adapt url with proxy settings
+# returns possibly modified url
+sub proxy_url
+{
+    my ($proxy_config, $url) = @_;
+
+    if ($url =~ /http/) {
+        my ($proxyhost, $proxyport, $proxytype) = @$proxy_config;
+        if ($proxyhost && $proxytype eq "reverse") {
+            if ($proxyport) {
+                $proxyhost .= ":$proxyport";
+            }
+            $url =~ s{(https?)://([^/]*)/}{$1://$proxyhost/};
+        }
+    }
+
+    return $url;
+}
 
 # Prints the header functions and definitions of the post_reboot
 # script.

From 57279c41909523f4052253408e8ce9956c877c32 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Tue, 5 Nov 2019 08:27:55 +0100
Subject: [PATCH 06/28] aii-ks: factor out repository proxy and baseurl
 modification

---
 aii-ks/src/main/perl/ks.pm | 78 ++++++++++++++++++++++++++------------
 1 file changed, 54 insertions(+), 24 deletions(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 5812cef9..26faea98 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -1333,30 +1333,70 @@ EOF
 }
 
 
+# create repo information with baseurl and proxy settings
+# return hashref with key the repo name
+sub get_repos
+{
+    my ($config) = @_;
+
+    my %res;
+
+    my $repos;
+    unless ( $config->elementExists(REPO) ) {
+      $this_app->error(REPO." not defined in configuration");
+      return
+    }
+    $repos = $config->getElement (REPO)->getTree();
+
+    my ($phost, $pport, $ptype) = proxy($config);
+
+    foreach my $repo (@$repos) {
+        if ($ptype && $ptype eq 'reverse') {
+            $repo->{protocols}->[0]->{url} =~ s{://[^/]*}{://$phost:$pport};
+        }
+
+        $repo->{baseurl} = $repo->{protocols}->[0]->{url};
+
+        # mandatory in 16.4 schema
+        #   these values are the default values in the schema
+        $repo->{enabled} = 1 if(! defined($repo->{enabled}));
+        $repo->{gpgcheck} = 0 if(! defined($repo->{gpgcheck}));
+
+        if (! $repo->{proxy} &&
+            $ptype &&
+            $ptype eq 'forward') {
+            $repo->{proxy} = "http://$phost:$pport/";
+        }
+
+        $res{$repo->{name}} = $repo;
+
+    }
+
+    return \%res;
+}
+
+
 sub yum_setup
 {
     my ($self, $config) = @_;
 
-    $self->debug(5,"Configuring YUM repositories...");
+    $self->debug(5, "Configuring YUM repositories...");
 
     # SPMA_OBSOLETES doesn't exist in 13.1 , assume false by default
     my $obsoletes = 0;
     if ( $config->elementExists(SPMA_OBSOLETES) ) {
         $obsoletes = $config->getElement (SPMA_OBSOLETES)->getTree();
     }
-    my $repos;
-    unless ( $config->elementExists(REPO) ) {
-      $this_app->error(REPO." not defined in configuration");
-      return
-    }
-    $repos = $config->getElement (REPO)->getTree();
+
+    my $repos = get_repos($config);
+    # error reported in get_repos
+    return if ! $repos;
 
     my $extra_yum_opts = {};
     if ( $config->elementExists(SPMA_YUMCONF) ) {
         $extra_yum_opts = $config->getElement (SPMA_YUMCONF)->getTree();
     }
 
-
     print <<EOF;
 mkdir -p /tmp/aii/yum/repos
 cat <<end_of_yum_conf > /tmp/aii/yum/yum.conf
@@ -1391,29 +1431,19 @@ end_of_yum_conf
 cat <<end_of_repos > /tmp/aii/yum/repos/aii.repo
 EOF
 
-    my ($phost, $pport, $ptype) = proxy($config);
-
-    $self->debug(5,"    Adding YUM repositories...");
+    $self->debug(5, "Adding YUM repositories...");
 
-    foreach my $repo (@$repos) {
-        if ($ptype && $ptype eq 'reverse') {
-            $repo->{protocols}->[0]->{url} =~ s{://[^/]*}{://$phost:$pport};
-        }
-        # mandatory in 16.4 schema
-        #   these values are the default values in the schema
-        $repo->{enabled} = 1 if(! defined($repo->{enabled}));
-        $repo->{gpgcheck} = 0 if(! defined($repo->{gpgcheck}));
+    foreach my $name (sort keys %$repos) {
+        my $repo = $repos->{$name};
 
         print <<EOF;
-[$repo->{name}]
-name=$repo->{name}
-baseurl=$repo->{protocols}->[0]->{url}
+[$name]
+name=$name
+baseurl=$repo->{baseurl}
 skip_if_unavailable=1
 EOF
         if ($repo->{proxy}) {
             print "proxy=$repo->{proxy}\n";
-        } elsif ($ptype && $ptype eq 'forward') {
-            print "proxy=http://$phost:$pport/\n";
         }
 
         # Handle inconsistent name mapping

From 7fa040918c31454fbbfce6b4217dd666f0a9c609 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 6 Nov 2019 13:57:56 +0100
Subject: [PATCH 07/28] aii-ks: generate the kickstart repo command from the
 SPMA repository configuration

---
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan |  3 ++
 aii-ks/src/main/perl/ks.pm                    | 42 ++++++++++++-------
 aii-ks/src/test/perl/kickstart_commands.t     |  3 ++
 .../src/test/perl/kickstart_packagesinpost.t  |  2 +-
 aii-ks/src/test/perl/kickstart_yum_setup.t    |  4 +-
 aii-ks/src/test/resources/kickstart.pan       | 21 ++++++++++
 .../src/test/resources/kickstart_bonding.pan  | 10 ++---
 .../src/test/resources/kickstart_commands.pan |  3 ++
 .../test/resources/kickstart_yum_setup.pan    | 22 ----------
 9 files changed, 67 insertions(+), 43 deletions(-)

diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index 5d1599d9..3f07620b 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -108,6 +108,9 @@ type structure_ks_ks_info = {
     "pre_install_script" ? type_absoluteURI
     "post_install_script" ? type_absoluteURI
     "post_reboot_script" ? type_absoluteURI
+    @{List of repositories (string in exact kickstart repo command syntax).
+      If a string starts with a '@', the repository is generated based on
+      the SPMA repositories with name(s) matching this pattern (without the leading '@').}
     "repo" ? string[]
     "timezone" : string
     @{NTP servers used by Anaconda}
diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 26faea98..38f0d710 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -434,12 +434,31 @@ timezone --utc $tree->{timezone}$ntp_servers
 rootpw --iscrypted $tree->{rootpw}
 EOF
 
+    my $repos = get_repos($config);
+    # error reported in get_repos
+    return if ! $repos;
+
     foreach my $url (@{$tree->{repo} || []}) {
         if ($url =~ m/^@(.+)$/) {
+            # find at least one repo with matching name
+            my $pattern = $1;
+            my @matches = grep {m/$pattern/} sort keys %$repos;
+            if (@matches) {
+                foreach my $reponame (@matches) {
+                    print "repo";
+                    foreach my $key (qw(name baseurl proxy includepkgs excludepkgs)) {
+                        my $val = $repos->{$reponame}->{$key};
+                        print " --$key=". (ref($val) eq 'ARRAY' ? join(',', @$val) : $val) if defined($val);
+                    }
+                    print "\n";
+                }
+            } else {
+                $this_app->error("kickstart repo: no spma repositories that match $pattern");
+            }
         } else {
             $url = proxy_url($proxy_config, $url);
+            print "repo $url\n";
         }
-        print "repo $url\n";
     }
 
     if ($tree->{cmdline}) {
@@ -549,8 +568,7 @@ EOF
     print "\n";
     print $version >= ANACONDA_VERSION_EL_6_0 ? '%end' : '', "\n";
 
-    return $unprocessed_packages;
-
+    return $unprocessed_packages, $repos;
 }
 
 # Writes the mountpoint definitions and LVM and MD settings
@@ -639,9 +657,9 @@ EOF
     # ends with the %postconfig section
     ksuserhooks ($config, ANACONDAHOOK);
 
-    my $packages = kscommands ($config);
+    my ($packages, $repos) = kscommands ($config);
 
-    return $packages;
+    return $packages, $repos;
 }
 
 # Create the action to be taken on the log files
@@ -1378,7 +1396,7 @@ sub get_repos
 
 sub yum_setup
 {
-    my ($self, $config) = @_;
+    my ($self, $config, $repos) = @_;
 
     $self->debug(5, "Configuring YUM repositories...");
 
@@ -1388,10 +1406,6 @@ sub yum_setup
         $obsoletes = $config->getElement (SPMA_OBSOLETES)->getTree();
     }
 
-    my $repos = get_repos($config);
-    # error reported in get_repos
-    return if ! $repos;
-
     my $extra_yum_opts = {};
     if ( $config->elementExists(SPMA_YUMCONF) ) {
         $extra_yum_opts = $config->getElement (SPMA_YUMCONF)->getTree();
@@ -1583,7 +1597,7 @@ EOF
 # this method.
 sub post_install_script
 {
-    my ($self, $config, $packages) = @_;
+    my ($self, $config, $packages, $repos) = @_;
 
     my $tree = $config->getElement (KS)->getTree;
     my $version = get_anaconda_version($tree);
@@ -1626,7 +1640,7 @@ EOF
         print "\n# Disable selinux via kernel parameter\ngrubby --update-kernel=DEFAULT --args=selinux=0\n";
     };
 
-    $self->yum_setup ($config);
+    $self->yum_setup ($config, $repos);
     $self->yum_install_packages ($config, $packages);
     ksuserscript ($config, POSTSCRIPT);
 
@@ -1807,9 +1821,9 @@ sub Configure
     }
 
     $self->ksopen ($config);
-    my $packages = $self->install ($config);
+    my ($packages, $repos) = $self->install ($config);
     $self->pre_install_script ($config);
-    $self->post_install_script ($config, $packages);
+    $self->post_install_script ($config, $packages, $repos);
     $self->ksclose;
     return 1;
 }
diff --git a/aii-ks/src/test/perl/kickstart_commands.t b/aii-ks/src/test/perl/kickstart_commands.t
index 0f41a230..b1678d40 100644
--- a/aii-ks/src/test/perl/kickstart_commands.t
+++ b/aii-ks/src/test/perl/kickstart_commands.t
@@ -39,6 +39,9 @@ like($fh, qr{^firewall\s--disabled }m, 'firwewall disabled present');
 like($fh, qr{^network\s--bootproto=dhcp}m, 'network dhcp present');
 like($fh, qr{^zerombr$}m, 'zerombr present');
 like($fh, qr{^services\s--disabled=disable1,DISABLE2\s--enabled=enable1,ENABLE2}m, "--dis/enable services present");
+like($fh, qr{^repo someurl}m, "repo as string");
+like($fh, qr{^repo --name=repo1 --baseurl=http://www.example.com --includepkgs=everything,else --excludepkgs=woo,hoo\*}m, "repo from pattern");
+unlike($fh, qr{^repo --name=repo0}m, "repo from pattern did not match other repo");
 
 like($fh, qr{^%packages\s--ignoremissing\s--resolvedeps\n^package\n^package2\nbind-utils\n}m, 'packages present');
 
diff --git a/aii-ks/src/test/perl/kickstart_packagesinpost.t b/aii-ks/src/test/perl/kickstart_packagesinpost.t
index 2af83b79..fd6fe75d 100644
--- a/aii-ks/src/test/perl/kickstart_packagesinpost.t
+++ b/aii-ks/src/test/perl/kickstart_packagesinpost.t
@@ -23,7 +23,7 @@ select($fh);
 my $ks = NCM::Component::ks->new('ks');
 my $cfg = get_config_for_profile('kickstart_packagesinpost');
 
-my $packref = NCM::Component::ks::kscommands($cfg);
+my ($packref, $repos) = NCM::Component::ks::kscommands($cfg);
 like($fh, qr{^%packages --ignoremissing --resolvedeps\n-notthispackage\n%end}m, "Only disabled packages in packages section");
 unlike($fh, qr{package2}, "No package2 in commands"); # one of the packages
 unlike($fh, qr{bind-utils}, "No bind-utils in commands"); # one of the auto-added packages
diff --git a/aii-ks/src/test/perl/kickstart_yum_setup.t b/aii-ks/src/test/perl/kickstart_yum_setup.t
index 2dab4254..0cc34032 100644
--- a/aii-ks/src/test/perl/kickstart_yum_setup.t
+++ b/aii-ks/src/test/perl/kickstart_yum_setup.t
@@ -25,7 +25,9 @@ select($fh);
 my $ks = NCM::Component::ks->new('ks');
 my $cfg = get_config_for_profile('kickstart_yum_setup');
 
-NCM::Component::ks::yum_setup($ks, $cfg);
+my $repos = NCM::Component::ks::get_repos($cfg);
+
+NCM::Component::ks::yum_setup($ks, $cfg, $repos);
 
 diag "$fh";
 
diff --git a/aii-ks/src/test/resources/kickstart.pan b/aii-ks/src/test/resources/kickstart.pan
index 3fb6a2dd..f9b64ae3 100644
--- a/aii-ks/src/test/resources/kickstart.pan
+++ b/aii-ks/src/test/resources/kickstart.pan
@@ -21,6 +21,27 @@ prefix "/software/packages";
 "{ncm-spma}/{14.2.1-1}/arch/noarch" = "";
 "{kernel-module-foo}" = nlist();
 
+prefix "/software/repositories/0";
+"name" = "repo0";
+"owner" = "me@example.com";
+"protocols/0/name" = "http";
+"protocols/0/url" = "http://www.example.com";
+"gpgcheck" = false;
+"repo_gpgcheck" = false;
+"gpgkey" = list(
+    "file:///path/to/key",
+    "https://somewhere/very/very/far",
+    "ftp://because/ftp/and/security/go/well/together",
+);
+"gpgcakey" = "file:///super/ca/key";
+
+prefix "/software/repositories/1";
+"name" = "repo1";
+"owner" = "me@example.com";
+"protocols/0/name" = "http";
+"protocols/0/url" = "http://www.example.com";
+"excludepkgs" = list('woo', 'hoo*');
+"includepkgs" = list('everything', 'else');
 
 # pxelinux and kickstart couple if bootproto is not dhcp
 prefix "/system/aii/nbp/pxelinux";
diff --git a/aii-ks/src/test/resources/kickstart_bonding.pan b/aii-ks/src/test/resources/kickstart_bonding.pan
index 7d9d0450..c8d8d576 100644
--- a/aii-ks/src/test/resources/kickstart_bonding.pan
+++ b/aii-ks/src/test/resources/kickstart_bonding.pan
@@ -1,11 +1,11 @@
-@{ 
-Profile to test kickstart bonding configuration 
+@{
+Profile to test kickstart bonding configuration
 @}
 object template kickstart_bonding;
 
 include 'kickstart';
 
-prefix "/system/network"; 
+prefix "/system/network";
 "interfaces/bond0/ip" = "1.2.3.0";
 "interfaces/bond0/netmask" = "255.255.255.0";
 "interfaces/bond0/bonding_opts" = nlist(
@@ -23,6 +23,6 @@ prefix "/system/network";
     );
 
 prefix "/system/aii/osinstall/ks";
-"bootproto" = "static"; 
+"bootproto" = "static";
 "version" = "13.21";
-"bonding" = true; 
\ No newline at end of file
+"bonding" = true;
\ No newline at end of file
diff --git a/aii-ks/src/test/resources/kickstart_commands.pan b/aii-ks/src/test/resources/kickstart_commands.pan
index cedb69e5..6e59cf68 100644
--- a/aii-ks/src/test/resources/kickstart_commands.pan
+++ b/aii-ks/src/test/resources/kickstart_commands.pan
@@ -5,3 +5,6 @@ object template kickstart_commands;
 
 include 'kickstart';
 
+prefix "/system/aii/osinstall/ks";
+"repo/0" = "someurl";
+"repo/1" = "@po1";  # should match repo1, not repo0
diff --git a/aii-ks/src/test/resources/kickstart_yum_setup.pan b/aii-ks/src/test/resources/kickstart_yum_setup.pan
index 51d324f6..4e0dc3db 100644
--- a/aii-ks/src/test/resources/kickstart_yum_setup.pan
+++ b/aii-ks/src/test/resources/kickstart_yum_setup.pan
@@ -5,28 +5,6 @@ object template kickstart_yum_setup;
 
 include 'kickstart';
 
-prefix "/software/repositories/0";
-"name" = "repo0";
-"owner" = "me@example.com";
-"protocols/0/name" = "http";
-"protocols/0/url" = "http://www.example.com";
-"gpgcheck" = false;
-"repo_gpgcheck" = false;
-"gpgkey" = list(
-    "file:///path/to/key",
-    "https://somewhere/very/very/far",
-    "ftp://because/ftp/and/security/go/well/together",
-);
-"gpgcakey" = "file:///super/ca/key";         
-    
-prefix "/software/repositories/1";
-"name" = "repo1";
-"owner" = "me@example.com";
-"protocols/0/name" = "http";
-"protocols/0/url" = "http://www.example.com";
-"excludepkgs" = list('woo', 'hoo*');
-"includepkgs" = list('everything', 'else');
-
 prefix "/software/components/spma/main_options";
 "exclude" = list("a", "b");
 "retries" = 40;

From 111240b361f1f8ba933c463eb8f93c8283dd3b37 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Mon, 18 Nov 2019 23:01:02 +0100
Subject: [PATCH 08/28] aii-ks: do not install kernel debug rpms in post

---
 aii-ks/src/main/perl/ks.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 38f0d710..cc2c8dbd 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -992,7 +992,7 @@ sub ksinstall_rpm
         $disabled = $config->getElement(DISABLED_REPOS)->getTree();
     }
     my $packager = $version >= ANACONDA_VERSION_EL_8_0 ? "dnf" : "yum";
-    my $cmd = "$packager -c /tmp/aii/yum/yum.conf -y install ";
+    my $cmd = "$packager -c /tmp/aii/yum/yum.conf -y -x 'kernel*debug*' install ";
 
     $cmd .= " --disablerepo=" . join(",", @$disabled) . " " if @$disabled;
 

From 9b486e1e25f7ed76f9748223f6bb6347c6060138 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 21 Nov 2019 13:34:21 +0100
Subject: [PATCH 09/28] aii-pxelinux: fix panlint line too long in schema

---
 aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
index d7f2be10..94a4bd0b 100644
--- a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
+++ b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
@@ -13,7 +13,8 @@ include 'pan/types';
 type structure_pxelinux_pxe_info = {
     "initrd" : string
     "kernel" : string
-    "ksdevice"  : string with match(SELF, '^(bootif|link)$') || is_hwaddr(SELF) || exists("/system/network/interfaces/" + escape(SELF))
+    "ksdevice"  : string with match(SELF, '^(bootif|link)$') || is_hwaddr(SELF) ||
+        exists("/system/network/interfaces/" + escape(SELF))
     "kslocation" : type_absoluteURI
     "label"  : string
     "append" ? string

From db9578ae7e654ab7d3317a0b3ffd25020b7c916b Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 7 Aug 2020 14:59:34 +0200
Subject: [PATCH 10/28] aii-ks: switch to globs for repository selection and
 some more code cleanup

---
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan |  3 +-
 aii-ks/src/main/perl/ks.pm                    | 53 ++++++++++---------
 aii-ks/src/test/perl/kickstart_proxy.t        |  8 ++-
 .../src/test/resources/kickstart_commands.pan |  6 +--
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index 3f07620b..1c9b106b 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -110,7 +110,8 @@ type structure_ks_ks_info = {
     "post_reboot_script" ? type_absoluteURI
     @{List of repositories (string in exact kickstart repo command syntax).
       If a string starts with a '@', the repository is generated based on
-      the SPMA repositories with name(s) matching this pattern (without the leading '@').}
+      the enabled SPMA repositories with name(s) matching this glob pattern (without the leading '@').
+      }
     "repo" ? string[]
     "timezone" : string
     @{NTP servers used by Anaconda}
diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index cc2c8dbd..b890645d 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -415,8 +415,8 @@ sub kscommands
     my @packages = @{$tree->{packages}};
     push(@packages, 'bind-utils'); # required for nslookup usage in ks-post-install
 
-    my $proxy_config = [proxy($config)];
-    my $installtype = proxy_url($proxy_config, $tree->{installtype});
+    my $proxy = proxy($config);
+    my $installtype = proxy_url($proxy, $tree->{installtype});
 
     my $ntp_servers = '';
     if ($tree->{ntpservers} && $version >= ANACONDA_VERSION_EL_7_0) {
@@ -441,10 +441,12 @@ EOF
     foreach my $url (@{$tree->{repo} || []}) {
         if ($url =~ m/^@(.+)$/) {
             # find at least one repo with matching name
-            my $pattern = $1;
-            my @matches = grep {m/$pattern/} sort keys %$repos;
+            my $glob_pattern = $1;
+            my @matches = match_glob($glob_pattern, sort keys %$repos);
             if (@matches) {
                 foreach my $reponame (@matches) {
+                    next if ! $repos->{$reponame}->{enabled};
+
                     print "repo";
                     foreach my $key (qw(name baseurl proxy includepkgs excludepkgs)) {
                         my $val = $repos->{$reponame}->{$key};
@@ -453,10 +455,10 @@ EOF
                     print "\n";
                 }
             } else {
-                $this_app->error("kickstart repo: no spma repositories that match $pattern");
+                $this_app->error("kickstart repo: no spma repositories that match $glob_pattern");
             }
         } else {
-            $url = proxy_url($proxy_config, $url);
+            $url = proxy_url($proxy, $url);
             print "repo $url\n";
         }
     }
@@ -997,13 +999,14 @@ sub ksinstall_rpm
     $cmd .= " --disablerepo=" . join(",", @$disabled) . " " if @$disabled;
 
     print $cmd, join("\\\n    ", @pkgs),
-         "|| fail 'Unable to install packages'\n";
+         " || fail 'Unable to install packages'\n";
 }
 
 sub proxy
 {
     my ($config) = @_;
-    my ($proxyhost, $proxyport, $proxytype);
+
+    my $proxy = {};
 
     my $spma = $config->getTree(SPMA);
     my $use_proxy = $spma->{proxy} || 0;
@@ -1015,36 +1018,37 @@ sub proxy
         my @proxies = split /,/, $tmp_proxyhost;
         if (scalar(@proxies) == 1) {
             # there's only one proxy specified
-            $proxyhost = $spma->{proxyhost};
+            $proxy->{host} = $spma->{proxyhost};
         } elsif (scalar(@proxies) > 1) {
             # optimize by picking the responding server as the proxy
             my $localhost = LOCALHOST;  # need a variable, not a constant
             my ($me) = grep { /\b$localhost\b/ } @proxies;
             $me ||= $proxies[0];
-            $proxyhost = $me;
+            $proxy->{host} = $me;
         }
+
         if ($spma->{proxyport}) {
-            $proxyport = $spma->{proxyport};
+            $proxy->{port} = $spma->{proxyport};
         }
         if ($spma->{proxytype}) {
-            $proxytype = $spma->{proxytype};
+            $proxy->{type} = $spma->{proxytype};
         }
     }
 
-    return ($proxyhost, $proxyport, $proxytype);
+    return $proxy;
 }
 
-# adapt url with proxy settings
+# adapt url with reverse proxy settings
 # returns possibly modified url
 sub proxy_url
 {
-    my ($proxy_config, $url) = @_;
+    my ($proxy, $url) = @_;
 
     if ($url =~ /http/) {
-        my ($proxyhost, $proxyport, $proxytype) = @$proxy_config;
-        if ($proxyhost && $proxytype eq "reverse") {
-            if ($proxyport) {
-                $proxyhost .= ":$proxyport";
+        if ($proxy->{host} && ($proxy->{type} || '') eq "reverse") {
+            my $proxyhost = $proxy->{host};
+            if ($proxy->{port}) {
+                $proxyhost .= ":$proxy->{port}";
             }
             $url =~ s{(https?)://([^/]*)/}{$1://$proxyhost/};
         }
@@ -1366,12 +1370,10 @@ sub get_repos
     }
     $repos = $config->getElement (REPO)->getTree();
 
-    my ($phost, $pport, $ptype) = proxy($config);
+    my $proxy = proxy($config);
 
     foreach my $repo (@$repos) {
-        if ($ptype && $ptype eq 'reverse') {
-            $repo->{protocols}->[0]->{url} =~ s{://[^/]*}{://$phost:$pport};
-        }
+        $repo->{protocols}->[0]->{url} = proxy_url($proxy, $repo->{protocols}->[0]->{url});
 
         $repo->{baseurl} = $repo->{protocols}->[0]->{url};
 
@@ -1381,9 +1383,8 @@ sub get_repos
         $repo->{gpgcheck} = 0 if(! defined($repo->{gpgcheck}));
 
         if (! $repo->{proxy} &&
-            $ptype &&
-            $ptype eq 'forward') {
-            $repo->{proxy} = "http://$phost:$pport/";
+            ($proxy->{type} || '') eq 'forward') {
+            $repo->{proxy} = "http://$proxy->{host}:$proxy->{port}/";
         }
 
         $res{$repo->{name}} = $repo;
diff --git a/aii-ks/src/test/perl/kickstart_proxy.t b/aii-ks/src/test/perl/kickstart_proxy.t
index 62d4bfef..7fdf4f99 100644
--- a/aii-ks/src/test/perl/kickstart_proxy.t
+++ b/aii-ks/src/test/perl/kickstart_proxy.t
@@ -25,14 +25,12 @@ use NCM::Component::ks;
 
 my $cfg = get_config_for_profile('kickstart_proxy');
 
-is_deeply([NCM::Component::ks::proxy($cfg)],
-          [qw(proxy.server2 1234 forward)],
+is_deeply(NCM::Component::ks::proxy($cfg),
+          {host => 'proxy.server2', port => 1234, type => 'forward'},
           "Return expected proxy config");
 
 $cfg = get_config_for_profile('kickstart_noproxy');
 
-is_deeply([NCM::Component::ks::proxy($cfg)],
-          [undef,undef,undef],
-          "undef/disabled proxy config");
+is_deeply(NCM::Component::ks::proxy($cfg), {}, "empty/disabled proxy config");
 
 done_testing();
diff --git a/aii-ks/src/test/resources/kickstart_commands.pan b/aii-ks/src/test/resources/kickstart_commands.pan
index 6e59cf68..d1b9b5aa 100644
--- a/aii-ks/src/test/resources/kickstart_commands.pan
+++ b/aii-ks/src/test/resources/kickstart_commands.pan
@@ -1,5 +1,5 @@
-@{ 
-Profile to ensure that the kickstart commands and packages section are generated 
+@{
+Profile to ensure that the kickstart commands and packages section are generated
 @}
 object template kickstart_commands;
 
@@ -7,4 +7,4 @@ include 'kickstart';
 
 prefix "/system/aii/osinstall/ks";
 "repo/0" = "someurl";
-"repo/1" = "@po1";  # should match repo1, not repo0
+"repo/1" = "@*po1*";  # should match repo1, not repo0

From 1f0788fa91e50036251c3dd9c98c7bf8a7f86830 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 7 Aug 2020 16:38:58 +0200
Subject: [PATCH 11/28] aii-ks: allow finegrained enable/disable/ignore control
 when generating the AII (post and post-reboot) repo configuration

---
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan |  6 +-
 aii-ks/src/main/perl/ks.pm                    | 75 ++++++++++++++++---
 aii-ks/src/test/perl/kickstart_yum_setup.t    | 41 ++++++++--
 .../src/test/resources/kickstart_yum_edi.pan  | 25 +++++++
 4 files changed, 129 insertions(+), 18 deletions(-)
 create mode 100644 aii-ks/src/test/resources/kickstart_yum_edi.pan

diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index 1c9b106b..ac3ce07a 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -122,8 +122,12 @@ type structure_ks_ks_info = {
     "ignoredisk" ? string[]
     @{Base packages needed for a Quattor client to run (CAF, CCM...)}
     "base_packages" : string[]
-    @{Repositories to disable while SPMA is not available}
+    @{Repositories to disable while SPMA is not available (evaluated as glob matching the repository name)}
     "disabled_repos" : string[] = list()
+    @{Repositories to enable while SPMA is not available (evaluated as glob matching the repository name)}
+    "enabled_repos" : string[] = list()
+    @{Repositories to ignore while SPMA is not available (evaluated as glob matching the repository name)}
+    "ignored_repos" : string[] = list()
     "packages_args" : string[] = list("--ignoremissing", "--resolvedeps")
     "end_script" ? string with {
         deprecated(0, "end_script is deprecated and will be removed in a future release");
diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index b890645d..d5faafd5 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -51,7 +51,6 @@ use constant { KS               => "/system/aii/osinstall/ks",
                NAMESERVER       => "/system/network/nameserver/0",
                FORWARDPROXY     => "forward",
                BASE_PKGS        => "/system/aii/osinstall/ks/base_packages",
-               DISABLED_REPOS   => "/system/aii/osinstall/ks/disabled_repos",
                LOCALHOST        => hostname(),
                INIT_SPMA_IGN_DEPS   => "/system/aii/osinstall/ks/init_spma_ignore_deps",
            };
@@ -988,16 +987,9 @@ sub ksinstall_rpm
     my $tree = $config->getElement(KS)->getTree;
     my $version = get_anaconda_version($tree);
 
-    # DISABLED_REPOS doesn't exist in 13.1
-    my $disabled = [];
-    if ( $config->elementExists(DISABLED_REPOS) ) {
-        $disabled = $config->getElement(DISABLED_REPOS)->getTree();
-    }
     my $packager = $version >= ANACONDA_VERSION_EL_8_0 ? "dnf" : "yum";
     my $cmd = "$packager -c /tmp/aii/yum/yum.conf -y -x 'kernel*debug*' install ";
 
-    $cmd .= " --disablerepo=" . join(",", @$disabled) . " " if @$disabled;
-
     print $cmd, join("\\\n    ", @pkgs),
          " || fail 'Unable to install packages'\n";
 }
@@ -1355,6 +1347,53 @@ EOF
 }
 
 
+# match based on length of glob
+# assuming the longest glob is the most specific
+# return list of (glob, action_value) sorted on decreasing length of glob
+#    -1: ignore
+#    0: disable
+#    1: enable
+sub make_enable_disable_ignore_repo_filter {
+    my ($config) = @_;
+
+    my $ks = $config->getTree(KS);
+
+    my %value_map = (
+        ignore => -1,
+        disable => 0,
+        enable => 1,
+    );
+
+    my %filter = map {$_ => ($ks->{$_."d_repos"} || [])} qw(enable disable ignore);
+
+    my @res = ();
+
+    foreach my $action (sort keys %filter) {
+        push(@res, map {[$_, $value_map{$action}]} @{$filter{$action}});
+    }
+
+    # reverse ordered length of glob sort
+    return [sort {length($b->[0]) <=> length($a->[0])} @res]
+};
+
+# first match of filter wins
+# return values
+#    undef: no match -> continue
+#    -1: ignore
+#    0: disable
+#    1: enable
+sub enable_disable_ignore_repo {
+    my ($name, $filter) = @_;
+
+    foreach my $op (@$filter) {
+        return $op->[1] if match_glob($op->[0], $name);
+    }
+
+    return undef;
+}
+
+
+
 # create repo information with baseurl and proxy settings
 # return hashref with key the repo name
 sub get_repos
@@ -1363,16 +1402,30 @@ sub get_repos
 
     my %res;
 
-    my $repos;
     unless ( $config->elementExists(REPO) ) {
       $this_app->error(REPO." not defined in configuration");
       return
     }
-    $repos = $config->getElement (REPO)->getTree();
+
+    my $repos = $config->getTree(REPO);
+
+    my $filter = make_enable_disable_ignore_repo_filter($config);
 
     my $proxy = proxy($config);
 
     foreach my $repo (@$repos) {
+        my $name = $repo->{name};
+        my $edi = enable_disable_ignore_repo($name, $filter);
+        if (defined($edi)) {
+            if ($edi == -1) {
+                $this_app->debug(5, "Ignore YUM repository $name");
+                next;
+            } else {
+                $this_app->debug(5, 'Force ', ($edi ? 'enable' : 'disable'), " YUM repository $name");
+                $repo->{enabled} = $edi;
+            }
+        }
+
         $repo->{protocols}->[0]->{url} = proxy_url($proxy, $repo->{protocols}->[0]->{url});
 
         $repo->{baseurl} = $repo->{protocols}->[0]->{url};
@@ -1387,7 +1440,7 @@ sub get_repos
             $repo->{proxy} = "http://$proxy->{host}:$proxy->{port}/";
         }
 
-        $res{$repo->{name}} = $repo;
+        $res{$name} = $repo;
 
     }
 
diff --git a/aii-ks/src/test/perl/kickstart_yum_setup.t b/aii-ks/src/test/perl/kickstart_yum_setup.t
index 0cc34032..eb5f2d23 100644
--- a/aii-ks/src/test/perl/kickstart_yum_setup.t
+++ b/aii-ks/src/test/perl/kickstart_yum_setup.t
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use Test::More;
-use Test::Quattor qw(kickstart_yum_setup);
+use Test::Quattor qw(kickstart_yum_setup kickstart_yum_edi);
 use Test::Quattor::RegexpTest;
 use NCM::Component::ks;
 use CAF::FileWriter;
@@ -18,15 +18,44 @@ Tests for the C<yum_setup> method.
 
 $CAF::Object::NoAction = 1;
 
+my $ks = NCM::Component::ks->new('ks');
+my $cfg = get_config_for_profile('kickstart_yum_edi');
+my $repos = NCM::Component::ks::get_repos($cfg);
+
+my $filter = NCM::Component::ks::make_enable_disable_ignore_repo_filter($cfg);
+diag "filter", explain $filter;
+is_deeply($filter,
+          [['disable*not', 1], ['disable*', 0], ['repo1', -1]],
+          "enable/disable/ignore filter");
+
+is(NCM::Component::ks::enable_disable_ignore_repo("disable_me_not", $filter), 1, "enable");
+is(NCM::Component::ks::enable_disable_ignore_repo("disable_me_now", $filter), 0, "disable");
+is(NCM::Component::ks::enable_disable_ignore_repo("repo1", $filter), -1, "ignore");
+ok(!defined(NCM::Component::ks::enable_disable_ignore_repo("repo2", $filter)), "continue");
+
+$repos = NCM::Component::ks::get_repos($cfg);
+diag "filtered repos", explain $repos;
+is_deeply({map {$_ => $repos->{$_}->{enabled}} keys %$repos},
+          {repo0 => 1, disable_me => 0, disable_me_not => 1},
+          "filtered enabled/disabled repos");
+
+# no filtering
+
+$cfg = get_config_for_profile('kickstart_yum_setup');
+$filter = NCM::Component::ks::make_enable_disable_ignore_repo_filter($cfg);
+diag "no filter", explain $filter;
+is_deeply($filter, [], "No enable/disable/ignore filter");
+
+$repos = NCM::Component::ks::get_repos($cfg);
+diag "unfiltered repos", explain $repos;
+is_deeply({map {$_ => $repos->{$_}->{enabled}} keys %$repos},
+          {repo0 => 1, repo1 => 1},
+          "unfiltered enabled/disabled repos");
+
 my $fh = CAF::FileWriter->new("target/test/ks_yum_setup");
 # This module simply prints to the default filehandle.
 select($fh);
 
-my $ks = NCM::Component::ks->new('ks');
-my $cfg = get_config_for_profile('kickstart_yum_setup');
-
-my $repos = NCM::Component::ks::get_repos($cfg);
-
 NCM::Component::ks::yum_setup($ks, $cfg, $repos);
 
 diag "$fh";
diff --git a/aii-ks/src/test/resources/kickstart_yum_edi.pan b/aii-ks/src/test/resources/kickstart_yum_edi.pan
new file mode 100644
index 00000000..50fbbc8f
--- /dev/null
+++ b/aii-ks/src/test/resources/kickstart_yum_edi.pan
@@ -0,0 +1,25 @@
+@{
+Profile to test enable/disable/ignore repos
+@}
+object template kickstart_yum_edi;
+
+include 'kickstart';
+
+prefix "/system/aii/osinstall/ks";
+"enabled_repos" = list("disable*not");
+"disabled_repos" = list("disable*");
+"ignored_repos" = list("repo1");
+
+prefix "/software/repositories/2";
+"name" = "disable_me";
+"enabled" = true;
+"owner" = "me@example.com";
+"protocols/0/name" = "http";
+"protocols/0/url" = "http://www.example.com";
+
+prefix "/software/repositories/3";
+"name" = "disable_me_not";
+"enabled" = false;
+"owner" = "me@example.com";
+"protocols/0/name" = "http";
+"protocols/0/url" = "http://www.example.com";

From 669c13ed62350f95df6e9adda20d60449f547296 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 14 Aug 2020 11:57:49 +0200
Subject: [PATCH 12/28] aii-ks: generalise the glob pattern used by repo
 selection and also enable it for installtype

---
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan |  10 +-
 aii-ks/src/main/perl/ks.pm                    | 107 ++++++++++++++----
 aii-ks/src/test/perl/kickstart_commands.t     |  21 +++-
 aii-ks/src/test/resources/kickstart.pan       |   2 +-
 .../src/test/resources/kickstart_commands.pan |   4 -
 .../resources/kickstart_commands_glob.pan     |  11 ++
 .../resources/regexps/kickstart_yum_setup     |   2 +-
 7 files changed, 121 insertions(+), 36 deletions(-)
 create mode 100644 aii-ks/src/test/resources/kickstart_commands_glob.pan

diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index ac3ce07a..4bdbfe32 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -90,6 +90,10 @@ type structure_ks_ks_info = {
     @{deprecated boolean. when defined, precedes value of mail/success.}
     "email_success" ? boolean with {deprecated(0, "email_success is deprecated; use mail/success instead"); true; }
     "firewall" ? structure_ks_ksfirewall
+    @{Kickstart installtype (string in exact kickstart repo command syntax).
+      If this contains a '@pattern@' substring, the installtype
+        (including the url and optional proxy option) is generated based on
+        the (first) enabled SPMA repository with name matching this glob pattern (without the '@').}
     "installtype" : string
     "installnumber" ? string
     "lang" : string = "en_US.UTF-8"
@@ -109,8 +113,10 @@ type structure_ks_ks_info = {
     "post_install_script" ? type_absoluteURI
     "post_reboot_script" ? type_absoluteURI
     @{List of repositories (string in exact kickstart repo command syntax).
-      If a string starts with a '@', the repository is generated based on
-      the enabled SPMA repositories with name(s) matching this glob pattern (without the leading '@').
+      If a string contains a '@pattern@' substring, the repository
+        (including the baseurl and optional proxy, includepkgs and exclude pkgs options)
+        is generated based on the enabled SPMA repositories
+        with name(s) matching this glob pattern (without the '@').
       }
     "repo" ? string[]
     "timezone" : string
diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index d5faafd5..7e454e03 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -17,7 +17,7 @@ our $EC = LC::Exception::Context->new->will_store_all;
 
 our $this_app = $main::this_app;
 # Modules that may be interesting for hooks.
-our @EXPORT_OK = qw (ksuserhooks ksinstall_rpm);
+our @EXPORT_OK = qw (ksuserhooks ksinstall_rpm get_repos replace_repo_glob);
 
 # PAN paths for some of the information needed to generate the
 # Kickstart.
@@ -414,8 +414,25 @@ sub kscommands
     my @packages = @{$tree->{packages}};
     push(@packages, 'bind-utils'); # required for nslookup usage in ks-post-install
 
+    my $repos = get_repos($config);
+    # error reported in get_repos
+    return if ! $repos;
+
     my $proxy = proxy($config);
-    my $installtype = proxy_url($proxy, $tree->{installtype});
+
+    my $installtype = $tree->{installtype};
+    my $proxy_noglob = sub {
+        my $txt = shift;
+        return [proxy_url($proxy, $txt)]
+    };
+    my $inst_msg = "installtype $installtype";
+    my $inst_globbed = replace_repo_glob($installtype, $repos, $proxy_noglob, 'url', {proxy => 'proxy'}, $inst_msg);
+    if (defined($inst_globbed)) {
+        $installtype = $inst_globbed->[0];
+    } else {
+        $this_app->error("$inst_msg glob had no matches");
+        return;
+    }
 
     my $ntp_servers = '';
     if ($tree->{ntpservers} && $version >= ANACONDA_VERSION_EL_7_0) {
@@ -433,32 +450,17 @@ timezone --utc $tree->{timezone}$ntp_servers
 rootpw --iscrypted $tree->{rootpw}
 EOF
 
-    my $repos = get_repos($config);
-    # error reported in get_repos
-    return if ! $repos;
+    my %repo_opt_map = map {$_ => $_} (qw(name proxy includepkgs excludepkgs));
 
     foreach my $url (@{$tree->{repo} || []}) {
-        if ($url =~ m/^@(.+)$/) {
-            # find at least one repo with matching name
-            my $glob_pattern = $1;
-            my @matches = match_glob($glob_pattern, sort keys %$repos);
-            if (@matches) {
-                foreach my $reponame (@matches) {
-                    next if ! $repos->{$reponame}->{enabled};
-
-                    print "repo";
-                    foreach my $key (qw(name baseurl proxy includepkgs excludepkgs)) {
-                        my $val = $repos->{$reponame}->{$key};
-                        print " --$key=". (ref($val) eq 'ARRAY' ? join(',', @$val) : $val) if defined($val);
-                    }
-                    print "\n";
-                }
-            } else {
-                $this_app->error("kickstart repo: no spma repositories that match $glob_pattern");
+        my $globbed = replace_repo_glob($url, $repos, $proxy_noglob, 'baseurl', \%repo_opt_map);
+        if (defined($globbed)) {
+            foreach my $newurl (@$globbed) {
+                print "repo $newurl\n";
             }
         } else {
-            $url = proxy_url($proxy, $url);
-            print "repo $url\n";
+            $this_app->error("repo url $url glob had no matches");
+            return;
         }
     }
 
@@ -1448,6 +1450,63 @@ sub get_repos
 }
 
 
+# Given a string $txt and hashref $repos, replace any occurence of glob @pattern@ with matching
+# repository converted in options. There can only be one glob.
+# $baseurl_key is the optionname that is prefixed to the glob (--<baseurl_key>=<repo{baseurl}>)
+#    unless it is not defined
+# $opt_map is the optional mapping to the generated text appended at the end: --<key>=$repo{<value>}
+# it returns a arrayref with all replaced text (empty list when there is a glob, but no repo was matched)
+# noglob is an anonymous sub that is called on the original $txt when there is no glob present
+#    this sub must return an arrayref
+# If only_one_txt is defined, the result is checked if there is exactly one, and the text is used
+#   to generate a warning
+# returns undef when there is no match
+sub replace_repo_glob
+{
+    my ($txt, $repos, $noglob, $baseurl_key, $opt_map, $only_one_txt) = @_;
+
+    my $res;
+
+    if ($txt =~ m/^([^@]*)@([^@]+)@([^@]*)$/) {
+        $res = [];
+
+        my $begin = $1;
+        my $glob_pattern = $2;
+        my $end = $3;
+
+        # find at least one repo with matching name
+        my @matches = match_glob($glob_pattern, sort keys %$repos);
+        if (@matches) {
+            foreach my $reponame (@matches) {
+                my $repo = $repos->{$reponame};
+                next if ! $repo->{enabled};
+
+                my $txt = (defined($baseurl_key) ? "--$baseurl_key=" : '').$repo->{baseurl};
+
+                my @opts;
+                foreach my $key (sort keys %{$opt_map || {}}) {
+                    my $val = $repo->{$opt_map->{$key}};
+                    push(@opts, "--$key=". (ref($val) eq 'ARRAY' ? join(',', @$val) : $val)) if defined($val);
+                }
+                push(@$res, join(' ', "$begin$txt$end", @opts));
+            }
+
+            if ($only_one_txt && (scalar @$res > 1)) {
+                $this_app->warn("$only_one_txt glob had more than one match.",
+                                "Only using first match. All matches: ", join('|', @$res));
+            };
+            $this_app->debug(5, "replace_repo_glob: pattern $glob_pattern matches ", join(',', @matches),
+                             " (from text $txt) with ", join('|', @$res));
+        } else {
+            $this_app->error("replace_repo_glob: no spma repositories that match $glob_pattern (from text $txt)");
+        }
+    } else {
+        $res = $noglob->($txt);
+    }
+
+    return $res;
+};
+
 sub yum_setup
 {
     my ($self, $config, $repos) = @_;
diff --git a/aii-ks/src/test/perl/kickstart_commands.t b/aii-ks/src/test/perl/kickstart_commands.t
index b1678d40..a8b04a74 100644
--- a/aii-ks/src/test/perl/kickstart_commands.t
+++ b/aii-ks/src/test/perl/kickstart_commands.t
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use Test::More;
-use Test::Quattor qw(kickstart_commands);
+use Test::Quattor qw(kickstart_commands kickstart_commands_glob);
 use NCM::Component::ks;
 use CAF::FileWriter;
 use CAF::Object;
@@ -39,13 +39,26 @@ like($fh, qr{^firewall\s--disabled }m, 'firwewall disabled present');
 like($fh, qr{^network\s--bootproto=dhcp}m, 'network dhcp present');
 like($fh, qr{^zerombr$}m, 'zerombr present');
 like($fh, qr{^services\s--disabled=disable1,DISABLE2\s--enabled=enable1,ENABLE2}m, "--dis/enable services present");
-like($fh, qr{^repo someurl}m, "repo as string");
-like($fh, qr{^repo --name=repo1 --baseurl=http://www.example.com --includepkgs=everything,else --excludepkgs=woo,hoo\*}m, "repo from pattern");
-unlike($fh, qr{^repo --name=repo0}m, "repo from pattern did not match other repo");
 
 like($fh, qr{^%packages\s--ignoremissing\s--resolvedeps\n^package\n^package2\nbind-utils\n}m, 'packages present');
 
 # close the selected FH and reset STDOUT
 NCM::Component::ks::ksclose;
 
+
+$fh = CAF::FileWriter->new("target/test/ks");
+select($fh);
+$cfg = get_config_for_profile('kickstart_commands_glob');
+NCM::Component::ks::kscommands($cfg);
+
+like($fh, qr{^url\s--url=http://www.example.com/some/extra/whatever\s--noverifyssl}m, 'installtype present (glob)');
+
+like($fh, qr{^repo someurl}m, "repo as string (no glob)");
+like($fh, qr{^repo --abc=def --baseurl=http://www.example1.com/weird --other=option --excludepkgs=woo,hoo\* --includepkgs=everything,else --name=repo1}m,
+     "repo from pattern (glob)");
+unlike($fh, qr{^repo --name=repo0}m, "repo from pattern did not match other repo (glob)");
+
+# close the selected FH and reset STDOUT
+NCM::Component::ks::ksclose;
+
 done_testing();
diff --git a/aii-ks/src/test/resources/kickstart.pan b/aii-ks/src/test/resources/kickstart.pan
index f9b64ae3..07ad1000 100644
--- a/aii-ks/src/test/resources/kickstart.pan
+++ b/aii-ks/src/test/resources/kickstart.pan
@@ -39,7 +39,7 @@ prefix "/software/repositories/1";
 "name" = "repo1";
 "owner" = "me@example.com";
 "protocols/0/name" = "http";
-"protocols/0/url" = "http://www.example.com";
+"protocols/0/url" = "http://www.example1.com";
 "excludepkgs" = list('woo', 'hoo*');
 "includepkgs" = list('everything', 'else');
 
diff --git a/aii-ks/src/test/resources/kickstart_commands.pan b/aii-ks/src/test/resources/kickstart_commands.pan
index d1b9b5aa..561a6b94 100644
--- a/aii-ks/src/test/resources/kickstart_commands.pan
+++ b/aii-ks/src/test/resources/kickstart_commands.pan
@@ -4,7 +4,3 @@ Profile to ensure that the kickstart commands and packages section are generated
 object template kickstart_commands;
 
 include 'kickstart';
-
-prefix "/system/aii/osinstall/ks";
-"repo/0" = "someurl";
-"repo/1" = "@*po1*";  # should match repo1, not repo0
diff --git a/aii-ks/src/test/resources/kickstart_commands_glob.pan b/aii-ks/src/test/resources/kickstart_commands_glob.pan
new file mode 100644
index 00000000..f6613c2c
--- /dev/null
+++ b/aii-ks/src/test/resources/kickstart_commands_glob.pan
@@ -0,0 +1,11 @@
+@{
+Profile to ensure that the kickstart commands and packages section are generated
+@}
+object template kickstart_commands_glob;
+
+include 'kickstart';
+
+prefix "/system/aii/osinstall/ks";
+"installtype" = "url @*epo0@/some/extra/whatever --noverifyssl";
+"repo/0" = "someurl";
+"repo/1" = "--abc=def @*po1*@/weird --other=option";  # should match repo1, not repo0
diff --git a/aii-ks/src/test/resources/regexps/kickstart_yum_setup b/aii-ks/src/test/resources/regexps/kickstart_yum_setup
index 3e1c221b..2af39af6 100644
--- a/aii-ks/src/test/resources/regexps/kickstart_yum_setup
+++ b/aii-ks/src/test/resources/regexps/kickstart_yum_setup
@@ -32,7 +32,7 @@ Test the yum_setup
 ^\s{4}ftp://because/ftp/and/security/go/well/together$
 ^\[repo1\]$
 ^name=repo1$
-^baseurl=http://www.example.com$
+^baseurl=http://www.example1.com$
 ^skip_if_unavailable=1$
 ^exclude=woo hoo\*$
 ^enabled=1$

From b1b6c25c532f521e8e62aaea0a4c3df7ab910a22 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 14 Aug 2020 22:47:05 +0200
Subject: [PATCH 13/28] aii-pxelinux: fix bug when calling _kernel_params with
 correct variant in _write_grub2_config

---
 aii-pxelinux/src/main/perl/pxelinux.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aii-pxelinux/src/main/perl/pxelinux.pm b/aii-pxelinux/src/main/perl/pxelinux.pm
index 7d31b61b..cff69c5f 100755
--- a/aii-pxelinux/src/main/perl/pxelinux.pm
+++ b/aii-pxelinux/src/main/perl/pxelinux.pm
@@ -491,7 +491,7 @@ sub _write_grub2_config
     $kernel_path =~ s{\bLOCALHOST\b}{LOCALHOST}e;
     $initrd_path =~ s{\bLOCALHOST\b}{LOCALHOST}e;
 
-    my @kernel_params = $self->_kernel_params($cfg, PXE_VARIANT_PXELINUX);
+    my @kernel_params = $self->_kernel_params($cfg, PXE_VARIANT_GRUB2);
     @kernel_params = () unless @kernel_params;
     my $kernel_params_text = join(' ', @kernel_params);
     $kernel_params_text = ' ' . $kernel_params_text if $kernel_params_text;

From 2da99b8ea471e6efbc537170eb5ca1e2e75d51fb Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 14 Aug 2020 22:51:00 +0200
Subject: [PATCH 14/28] aii-pxelinux: add globbing to the pxelinux initrd and
 kernel paths

---
 .../main/pan/quattor/aii/pxelinux/schema.pan  |  8 ++-
 aii-pxelinux/src/main/perl/pxelinux.pm        | 54 +++++++++++++------
 .../src/test/perl/NCM/Component/ks.pm         | 17 +++++-
 .../src/test/perl/pxelinux_ks_logging.t       |  4 +-
 4 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
index 94a4bd0b..9d8ccf42 100644
--- a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
+++ b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
@@ -11,8 +11,14 @@ include 'pan/types';
     PXE configuration
 }
 type structure_pxelinux_pxe_info = {
-    "initrd" : string
+    @{Kernel path (string in exact syntax).
+      If this contains a '@pattern@' substring, the kernel path is generated based on
+        the (first) enabled SPMA repository with name matching this glob pattern (without the '@').}
     "kernel" : string
+    @{Initrd path (string in exact syntax).
+      If this contains a '@pattern@' substring, the initrd path is generated based on
+        the (first) enabled SPMA repository with name matching this glob pattern (without the '@').}
+    "initrd" : string
     "ksdevice"  : string with match(SELF, '^(bootif|link)$') || is_hwaddr(SELF) ||
         exists("/system/network/interfaces/" + escape(SELF))
     "kslocation" : type_absoluteURI
diff --git a/aii-pxelinux/src/main/perl/pxelinux.pm b/aii-pxelinux/src/main/perl/pxelinux.pm
index cff69c5f..15ce6e00 100755
--- a/aii-pxelinux/src/main/perl/pxelinux.pm
+++ b/aii-pxelinux/src/main/perl/pxelinux.pm
@@ -3,7 +3,7 @@
 use Sys::Hostname;
 use CAF::FileWriter;
 use CAF::Object qw(SUCCESS CHANGED);
-use NCM::Component::ks qw (ksuserhooks);
+use NCM::Component::ks qw (ksuserhooks get_repos replace_repo_glob);
 use File::stat;
 use File::Basename qw(dirname);
 use Time::localtime;
@@ -310,7 +310,7 @@ sub _pxe_network_bonding {
 # create a list with all kernel parameters for the kickstart installation
 sub _kernel_params_ks
 {
-    my ($self, $cfg, $variant) = @_;
+    my ($self, $cfg, $variant, $initrd_path) = @_;
 
     my $pxe_config = $cfg->getElement (PXEROOT)->getTree;
 
@@ -340,9 +340,7 @@ sub _kernel_params_ks
     # with previous AII versions for easier comparisons.
     my @kernel_params =  ("ramdisk=32768");
     if ($variant == PXE_VARIANT_PXELINUX) {
-        my $initrd = $pxe_config->{initrd};
-        $initrd =~ s{\bLOCALHOST\b}{LOCALHOST}e;
-        push @kernel_params, "initrd=$initrd";
+        push @kernel_params, "initrd=$initrd_path";
     }
     push (@kernel_params, "${keyprefix}ks=$ksloc");
 
@@ -421,10 +419,10 @@ sub _kernel_params_ks
 # create a list with all required kernel parameters, based on the configuration
 sub _kernel_params
 {
-    my ($self, $cfg, $variant) = @_;
+    my ($self, $cfg, $variant, $initrd_path) = @_;
 
     if ($cfg->elementExists(KS)) {
-        return $self->_kernel_params_ks($cfg, $variant);
+        return $self->_kernel_params_ks($cfg, $variant, $initrd_path);
     } else {
         # Non-linux hosts may use pxelinux to chain-load their own bootloader
         $self->debug (1, "No Kickstart-related parameters in configuration: no kernel parameters added.");
@@ -432,6 +430,34 @@ sub _kernel_params
     }
 }
 
+sub _kernel_initrd_path
+{
+    my ($self, $cfg, $prefix) = @_;
+
+    my $repos = get_repos($cfg);
+
+    my $localhost_noglob = sub {
+        my $txt = shift;
+        $txt =~ s{\bLOCALHOST\b}{LOCALHOST}e;
+        return [$txt];
+    };
+
+    my $pxe_config = $cfg->getTree(PXEROOT);
+
+    my @res;
+    foreach my $key (qw(kernel initrd)) {
+        my $val = $pxe_config->{$key};
+        my $globbed = replace_repo_glob($prefix.$val, $repos, $localhost_noglob, undef, undef, "$key $val");
+        if (!defined($globbed)) {
+            $this_app->error("$key $val glob had no matches");
+            return;
+        }
+        push(@res, $globbed->[0]);
+    }
+
+    return @res;
+};
+
 # Write the PXELINUX configuration file.
 sub _write_pxelinux_config
 {
@@ -440,13 +466,12 @@ sub _write_pxelinux_config
     my $fh = CAF::FileWriter->open ($self->_file_path ($cfg, PXE_VARIANT_PXELINUX),
                     log => $self, mode => 0644);
 
+    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, "");
+
     my $appendtxt = '';
-    my @appendoptions = $self->_kernel_params($cfg, PXE_VARIANT_PXELINUX);
+    my @appendoptions = $self->_kernel_params($cfg, PXE_VARIANT_PXELINUX, $initrd_path);
     $appendtxt = join(" ", "append", @appendoptions) if @appendoptions;
 
-    my $kernel = $pxe_config->{kernel};
-    $kernel =~ s{\bLOCALHOST\b}{LOCALHOST}e;
-
     my $entry_label = "Install $pxe_config->{label}";
     print $fh <<EOF;
 # File generated by pxelinux AII plug-in.
@@ -454,7 +479,7 @@ sub _write_pxelinux_config
 default $entry_label
 
 label $entry_label
-    kernel $kernel
+    kernel $kernel_path
     $appendtxt
 EOF
 
@@ -485,11 +510,8 @@ sub _write_grub2_config
     };
     my $kernel_root = $this_app->option(GRUB2_EFI_KERNEL_ROOT);
     $kernel_root = '' unless defined($kernel_root);
-    my $kernel_path = "$kernel_root/$pxe_config->{kernel}";
-    my $initrd_path = "$kernel_root/$pxe_config->{initrd}";
 
-    $kernel_path =~ s{\bLOCALHOST\b}{LOCALHOST}e;
-    $initrd_path =~ s{\bLOCALHOST\b}{LOCALHOST}e;
+    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, "$kernel_root/");
 
     my @kernel_params = $self->_kernel_params($cfg, PXE_VARIANT_GRUB2);
     @kernel_params = () unless @kernel_params;
diff --git a/aii-pxelinux/src/test/perl/NCM/Component/ks.pm b/aii-pxelinux/src/test/perl/NCM/Component/ks.pm
index d94decdc..959ca608 100644
--- a/aii-pxelinux/src/test/perl/NCM/Component/ks.pm
+++ b/aii-pxelinux/src/test/perl/NCM/Component/ks.pm
@@ -14,7 +14,7 @@ use strict;
 use warnings;
 use parent qw(Exporter);
 
-our @EXPORT_OK = qw(ksuserhooks get_ks_userhook_args);
+our @EXPORT_OK = qw(ksuserhooks get_ks_userhook_args get_repos replace_repo_glob);
 
 my $_args;
 
@@ -25,4 +25,19 @@ sub get_ks_userhook_args {
 
 sub ksuserhooks { $_args = \@_; }
 
+sub get_repos
+{
+    my ($config) = @_;
+
+    return {};
+}
+
+
+sub replace_repo_glob
+{
+    my ($txt, $repos, $noglob, $baseurl_key, $opt_map, $only_one_txt) = @_;
+
+    return $noglob->($txt);
+}
+
 1;
diff --git a/aii-pxelinux/src/test/perl/pxelinux_ks_logging.t b/aii-pxelinux/src/test/perl/pxelinux_ks_logging.t
index e2dcbf9e..c8cfea32 100644
--- a/aii-pxelinux/src/test/perl/pxelinux_ks_logging.t
+++ b/aii-pxelinux/src/test/perl/pxelinux_ks_logging.t
@@ -51,7 +51,7 @@ for my $variant_constant (@PXE_VARIANTS) {
 
     $comp->$config_method($cfg);
     my $fh = get_file($fp);
-    
+
     like($fh, qr{^\s{4}$kernel_params_cmd\s.*?\ssyslog=logserver:514\sloglevel=debug(\s|$)}m, "append line (variant=$variant_name)");
 };
 
@@ -67,7 +67,7 @@ for my $variant_constant (@PXE_VARIANTS) {
 
     $comp->$config_method($cfg);
     my $fh = get_file($fp);
-    
+
     unlike($fh, qr{\ssyslog}, "no syslog config (variant=$variant_name)");
     unlike($fh, qr{\sloglevel}, "no loglevel config (variant=$variant_name)");
 };

From 3928f1c10f36cdb44664f4ff12848a3510ee49ae Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 19 Aug 2020 00:28:32 +0200
Subject: [PATCH 15/28] aii-ks: handle disabled/ignored packages in post too

---
 aii-ks/src/main/perl/ks.pm                      | 17 ++++++++++++-----
 aii-ks/src/test/perl/kickstart_packagesinpost.t |  8 ++++++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 7e454e03..32ceb6cc 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -558,9 +558,15 @@ EOF
     my @packages_in_packages = @packages;
     if ($tree->{packagesinpost}) {
         # to be installed later in %post using all repos
-        # disabled/ignored packages cannot be handled in packagesinpost
+        # disabled/ignored packages can be handled in packagesinpost (at least in EL7+),
+        # but better make sure they are not pulled in via some other method, so adding them here as well
         my $pattern = '^-';
         @packages_in_packages = grep {m/$pattern/} @packages;
+
+        # for EL7+, use never matching pattern, so all packages are also carried over
+        # to the %post section (incl the ones starting with a -)
+        $pattern = '$^' if $version >= ANACONDA_VERSION_EL_7_0;
+
         push(@$unprocessed_packages, grep {$_ !~ m/$pattern/} @packages);
     }
 
@@ -990,10 +996,11 @@ sub ksinstall_rpm
     my $version = get_anaconda_version($tree);
 
     my $packager = $version >= ANACONDA_VERSION_EL_8_0 ? "dnf" : "yum";
-    my $cmd = "$packager -c /tmp/aii/yum/yum.conf -y -x 'kernel*debug*' install ";
-
-    print $cmd, join("\\\n    ", @pkgs),
-         " || fail 'Unable to install packages'\n";
+    print join("\\\n    ",
+               "$packager -c /tmp/aii/yum/yum.conf -y install",
+               (map {s/^-//; "-x '$_'"} grep {$_ =~ /^-/} @pkgs),
+               (grep {$_ !~ /^-/} @pkgs)
+        ), " || fail 'Unable to install packages'\n";
 }
 
 sub proxy
diff --git a/aii-ks/src/test/perl/kickstart_packagesinpost.t b/aii-ks/src/test/perl/kickstart_packagesinpost.t
index fd6fe75d..839fa4b1 100644
--- a/aii-ks/src/test/perl/kickstart_packagesinpost.t
+++ b/aii-ks/src/test/perl/kickstart_packagesinpost.t
@@ -24,15 +24,19 @@ my $ks = NCM::Component::ks->new('ks');
 my $cfg = get_config_for_profile('kickstart_packagesinpost');
 
 my ($packref, $repos) = NCM::Component::ks::kscommands($cfg);
+
+diag "kscommands packref ", explain $packref, " repos ", explain $repos, "\n$fh";
 like($fh, qr{^%packages --ignoremissing --resolvedeps\n-notthispackage\n%end}m, "Only disabled packages in packages section");
 unlike($fh, qr{package2}, "No package2 in commands"); # one of the packages
 unlike($fh, qr{bind-utils}, "No bind-utils in commands"); # one of the auto-added packages
 
+
 $ks->yum_install_packages($cfg, $packref);
+diag "yum_install_packages\n$fh";
 like($fh, qr{\spackage2}, "package2 added"); # one of the packages
 like($fh, qr{\sbind-utils}, "bind-utils added"); # one of the auto-added packages
-unlike($fh, qr{\snotthispackage},
-       "disabled/ignored packages are not added to the package install in post");
+like($fh, qr{-x\s'notthispackage'},
+       "disabled/ignored packages are added to the package install (again) in post");
 
 # close the selected FH and reset STDOUT
 NCM::Component::ks::ksclose;

From eb9c80a917916a79c57903040462844a2b859ea6 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 19 Aug 2020 00:29:04 +0200
Subject: [PATCH 16/28] aii-ks: add boolean to control installation of (exact)
 kernel packages in post

---
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan |  2 ++
 aii-ks/src/main/perl/ks.pm                    | 23 +++++++++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index 4bdbfe32..4d4ba258 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -149,6 +149,8 @@ type structure_ks_ks_info = {
     @{agree with EULA (EL7+)}
     'eula' ? boolean
     'packagesinpost' ? boolean
+    @{install the correct kernel rpms as defined in /software/packages (if any)}
+    'kernelinpost' : boolean = true
     @{configure bonding (when not defined, it will be tried best-effort depending on OS version and configuration)}
     'bonding' ? boolean
     'lvmforce' ? boolean
diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 32ceb6cc..59d8f948 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -50,7 +50,6 @@ use constant { KS               => "/system/aii/osinstall/ks",
                CCM_CONFIG_PATH  => "/software/components/ccm",
                NAMESERVER       => "/system/network/nameserver/0",
                FORWARDPROXY     => "forward",
-               BASE_PKGS        => "/system/aii/osinstall/ks/base_packages",
                LOCALHOST        => hostname(),
                INIT_SPMA_IGN_DEPS   => "/system/aii/osinstall/ks/init_spma_ignore_deps",
            };
@@ -1678,12 +1677,18 @@ sub yum_install_packages
 {
     my ($self, $config, $packages) = @_;
 
-    $self->debug(5,"Adding packages to install with YUM...");
+    $self->debug(5, "Adding packages to install with YUM...");
 
     my @pkgs;
-    my $t = $config->getElement (PKG)->getTree();
+    my $pkgtree = $config->getTree(PKG);
+    my $ks = $config->getTree(KS);
+
+    my %base = map(($_ => 1), @{$ks->{base_packages}});
+
 
-    my %base = map(($_ => 1), @{$config->getElement (BASE_PKGS)->getTree()});
+    my @install = ("ncm-spma", "ncm-grub");
+    push(@install, "kernel") if $ks->{kernelinpost};
+    my $pattern = '^('.join('|', @install).')';
 
     print <<EOF;
 # This one will be reinstalled by Yum in the correct version for our
@@ -1693,10 +1698,10 @@ rpm -e --nodeps kernel-firmware
 # it's needed at all, it will be reinstalled by the SPMA component.
 rpm -e --nodeps yum-conf
 EOF
-    foreach my $pkg (sort keys %$t) {
+    foreach my $pkg (sort keys %$pkgtree) {
         my $pkgst = unescape($pkg);
-        if ($pkgst =~ m{^(kernel|ncm-spma|ncm-grub)} || exists($base{$pkgst})) {
-            push (@pkgs, [$pkgst, $t->{$pkg}]);
+        if ($pkgst =~ m{$pattern} || exists($base{$pkgst})) {
+            push (@pkgs, [$pkgst, $pkgtree->{$pkg}]);
         }
     }
 
@@ -1722,9 +1727,9 @@ sub post_install_script
     my $tree = $config->getElement (KS)->getTree;
     my $version = get_anaconda_version($tree);
 
-    $self->debug(5,"Adding postinstall script...");
+    $self->debug(5, "Adding postinstall script...");
 
-    my $logfile='/tmp/post-log.log';
+    my $logfile = '/tmp/post-log.log';
     my $logaction = log_action($config, $logfile);
 
     print <<EOF;

From a5733f0653dfae1d02d41c1e25cc1b032e31ac65 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 19 Aug 2020 23:59:09 +0200
Subject: [PATCH 17/28] aii-ks: only root access to /tmp yum and log files

---
 aii-ks/src/main/perl/ks.pm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 59d8f948..668012bc 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -1532,6 +1532,8 @@ sub yum_setup
 
     print <<EOF;
 mkdir -p /tmp/aii/yum/repos
+chmod 700 /tmp/aii
+
 cat <<end_of_yum_conf > /tmp/aii/yum/yum.conf
 [main]
 EOF
@@ -1750,6 +1752,9 @@ EOF
 # %post phase. The base system has already been installed. Let's do
 # some minor changes and prepare it for being configured.
 $logaction
+
+chmod 600 $logfile
+
 echo 'Begin of post section'
 set -x
 

From e0bf6b693457f349f090862a52ba642d6fb55404 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Tue, 16 Mar 2021 15:39:14 +0100
Subject: [PATCH 18/28] aii-pxelinux: Fix UEFI boot over http

---
 aii-core/src/main/perl/Shellfe.pm             |  3 ++
 aii-pxelinux/src/main/perl/pxelinux.pm        | 30 +++++++++++++++----
 .../src/test/perl/write_grub2_config.t        | 20 ++++++++++---
 .../test/resources/pxelinux_config_common.pan | 15 +++++-----
 .../src/test/resources/pxelinux_grub2.pan     | 13 ++++++++
 .../src/test/resources/pxelinux_grub_glob.pan | 14 +++++++++
 6 files changed, 78 insertions(+), 17 deletions(-)
 create mode 100644 aii-pxelinux/src/test/resources/pxelinux_grub2.pan
 create mode 100644 aii-pxelinux/src/test/resources/pxelinux_grub_glob.pan

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index 1244fbef..80f9f44f 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -389,6 +389,9 @@ sub _initialize
             $kernel_root = '' if ( $kernel_root eq '/' );
             $self->{CONFIG}->set(GRUB2_EFI_KERNEL_ROOT, $kernel_root);
         }
+    } else {
+        $self->{CONFIG}->set(GRUB2_EFI_KERNEL_ROOT, undef)
+            if $self->option(GRUB2_EFI_KERNEL_ROOT) eq NBPDIR_VARIANT_DISABLED;
     }
     # GRUB2_EFI_INITRD_CMD is always derived from GRUB2_EFI_LINUX_CMD as
     # Grub2 has a set of linux/initrd command pairs that must match together.
diff --git a/aii-pxelinux/src/main/perl/pxelinux.pm b/aii-pxelinux/src/main/perl/pxelinux.pm
index 15ce6e00..74593acb 100755
--- a/aii-pxelinux/src/main/perl/pxelinux.pm
+++ b/aii-pxelinux/src/main/perl/pxelinux.pm
@@ -436,6 +436,9 @@ sub _kernel_initrd_path
 
     my $repos = get_repos($cfg);
 
+    use Data::Dumper;
+    $self->debug(1, "repos ".Dumper($repos));
+
     my $localhost_noglob = sub {
         my $txt = shift;
         $txt =~ s{\bLOCALHOST\b}{LOCALHOST}e;
@@ -447,12 +450,29 @@ sub _kernel_initrd_path
     my @res;
     foreach my $key (qw(kernel initrd)) {
         my $val = $pxe_config->{$key};
-        my $globbed = replace_repo_glob($prefix.$val, $repos, $localhost_noglob, undef, undef, "$key $val");
+        # assuming the prefix contains no part of the globs
+        my $globbed = replace_repo_glob($val, $repos, $localhost_noglob, undef, undef, "$key $val");
         if (!defined($globbed)) {
             $this_app->error("$key $val glob had no matches");
             return;
         }
-        push(@res, $globbed->[0]);
+
+        my $res = $globbed->[0];
+
+        if (defined($prefix)) {
+            # aka EFI
+            if ($res =~ m{^(http(?:s)?)://([^/]+)/(.*)$}) {
+                # typically from the glob, as this is not a valid kernel/initrd path
+                $res = "($1,$2)/$3";
+            } else {
+                # Avoid having an uncoditional "/" at the beginning, because that would
+                # break if the kernel/initrd location uses the "(http,XXXX)/..." or
+                # "(tftp,XXXX)/..." syntax
+                $res = ($res =~ m/^\((http|tftp),/ ? "" : $prefix) . $res;
+            };
+        };
+
+        push(@res, $res);
     }
 
     return @res;
@@ -466,7 +486,8 @@ sub _write_pxelinux_config
     my $fh = CAF::FileWriter->open ($self->_file_path ($cfg, PXE_VARIANT_PXELINUX),
                     log => $self, mode => 0644);
 
-    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, "");
+    # pass undef, this is not EFI
+    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, undef);
 
     my $appendtxt = '';
     my @appendoptions = $self->_kernel_params($cfg, PXE_VARIANT_PXELINUX, $initrd_path);
@@ -509,9 +530,8 @@ sub _write_grub2_config
         return 0;
     };
     my $kernel_root = $this_app->option(GRUB2_EFI_KERNEL_ROOT);
-    $kernel_root = '' unless defined($kernel_root);
 
-    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, "$kernel_root/");
+    my ($kernel_path, $initrd_path) = $self->_kernel_initrd_path($cfg, defined($kernel_root) ? "$kernel_root/" : "");
 
     my @kernel_params = $self->_kernel_params($cfg, PXE_VARIANT_GRUB2);
     @kernel_params = () unless @kernel_params;
diff --git a/aii-pxelinux/src/test/perl/write_grub2_config.t b/aii-pxelinux/src/test/perl/write_grub2_config.t
index a3b16ff0..931d8d79 100644
--- a/aii-pxelinux/src/test/perl/write_grub2_config.t
+++ b/aii-pxelinux/src/test/perl/write_grub2_config.t
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use Test::More;
-use Test::Quattor qw(pxelinux_base_config);
+use Test::Quattor qw(pxelinux_base_config pxelinux_grub2 pxelinux_grub_glob);
 use NCM::Component::PXELINUX::constants qw(:pxe_variants :pxe_constants);
 use NCM::Component::pxelinux;
 use CAF::FileWriter;
@@ -36,16 +36,17 @@ sub check_config {
 
     # Retrieve config file name matching the configuration
     my $fp = $comp->_file_path($cfg, PXE_VARIANT_GRUB2);
-    
+
     # Check config file contents
     my $fh = get_file($fp);
     my $hostname = hostname();
+    my $prefix = $kernel_root ? "$kernel_root/" : "";
     like($fh, qr{^set default=0$}m, "default kernel ($test_msg)");
     like($fh, qr{^set timeout=\d+$}m, "Grub2 menu timeout ($test_msg)");
     like($fh, qr(^menuentry\s"Install\s[\w\-\s()\[\]]+"\s\{$)m, "Grub2 menu entry ($test_msg)");
     like($fh, qr{^\s{4}set root=\(pxe\)$}m, "Grub2 root ($test_msg)");
-    like($fh, qr{^\s{4}$TEST_EFI_LINUX_CMD $kernel_root/mykernel}m, "Kernel loading ($test_msg)");
-    like($fh, qr{^\s{4}$test_efi_initrd_cmd $kernel_root/path/to/initrd$}m, "initrd loading ($test_msg)");
+    like($fh, qr{^\s{4}$TEST_EFI_LINUX_CMD ${prefix}mykernel}m, "Kernel loading ($test_msg)");
+    like($fh, qr{^\s{4}$test_efi_initrd_cmd ${prefix}path/to/initrd$}m, "initrd loading ($test_msg)");
     like($fh, qr(^})m, "end of menu entry ($test_msg)");
 }
 
@@ -68,4 +69,15 @@ $this_app->{CONFIG}->define(GRUB2_EFI_KERNEL_ROOT);
 $this_app->{CONFIG}->set(GRUB2_EFI_KERNEL_ROOT, $GRUB2_EFI_KERNEL_ROOT_VALUE);
 check_config($comp, $cfg, $GRUB2_EFI_KERNEL_ROOT_VALUE, 'No GRUB2_EFI_KERNEL_ROOT');
 
+$cfg = get_config_for_profile('pxelinux_grub2');
+
+$this_app->{CONFIG}->set(GRUB2_EFI_KERNEL_ROOT, "/foo/bar");
+check_config($comp, $cfg, qr{\(http,myhost\.example\)}, 'Profile ignores GRUB2_EFI_KERNEL_ROOT');
+
+$cfg = get_config_for_profile('pxelinux_grub_glob');
+
+$this_app->{CONFIG}->set(GRUB2_EFI_KERNEL_ROOT, "/foooo/baarr");
+check_config($comp, $cfg, qr{\(http,abc.def\)},
+             'Profile ignores GRUB2_EFI_KERNEL_ROOT, and replaces protocol');
+
 done_testing();
diff --git a/aii-pxelinux/src/test/resources/pxelinux_config_common.pan b/aii-pxelinux/src/test/resources/pxelinux_config_common.pan
index eb54b2ce..0282a8d6 100644
--- a/aii-pxelinux/src/test/resources/pxelinux_config_common.pan
+++ b/aii-pxelinux/src/test/resources/pxelinux_config_common.pan
@@ -14,18 +14,18 @@ prefix "/system/network";
 "nameserver/0" = 'nm1';
 "nameserver/1" = 'nm2';
 "default_gateway" = "133.2.85.1";
-"interfaces/eth0" = nlist("ip", "133.2.85.234",
-                          "netmask", "255.255.255.0",
-                         );
-"interfaces/eth1" = nlist("onboot", "no",
-                         );
-
+"interfaces/eth0" = dict(
+    "ip", "133.2.85.234",
+    "netmask", "255.255.255.0",
+    );
+"interfaces/eth1" = dict(
+    "onboot", "no",
+    );
 
 prefix "/hardware/cards/nic";
 "eth0/hwaddr" = "00:11:22:33:44:55";
 "eth1/hwaddr" = "00:11:22:33:44:66";
 
-
 prefix "/system/aii/nbp/pxelinux";
 "initrd" = "path/to/initrd";
 "kernel" = 'mykernel';
@@ -35,4 +35,3 @@ prefix "/system/aii/nbp/pxelinux";
 "firmware" = "firmware.cfg";
 "livecd" = "livecd.cfg";
 "rescue" = "rescue.cfg";
-
diff --git a/aii-pxelinux/src/test/resources/pxelinux_grub2.pan b/aii-pxelinux/src/test/resources/pxelinux_grub2.pan
new file mode 100644
index 00000000..f8f72616
--- /dev/null
+++ b/aii-pxelinux/src/test/resources/pxelinux_grub2.pan
@@ -0,0 +1,13 @@
+@{
+Grub2 object template for aii-pxelinux unit tests.
+Only include pxelinux_config.common.pan
+}
+
+object template pxelinux_grub2;
+
+include 'pxelinux_config_common';
+
+prefix "/system/aii/nbp/pxelinux";
+
+"kernel" = "(http,myhost.example)/mykernel";
+"initrd" = "(http,myhost.example)/path/to/initrd";
diff --git a/aii-pxelinux/src/test/resources/pxelinux_grub_glob.pan b/aii-pxelinux/src/test/resources/pxelinux_grub_glob.pan
new file mode 100644
index 00000000..1da2030b
--- /dev/null
+++ b/aii-pxelinux/src/test/resources/pxelinux_grub_glob.pan
@@ -0,0 +1,14 @@
+@{
+Grub2 object template for aii-pxelinux unit tests.
+Only include pxelinux_config.common.pan
+}
+
+object template pxelinux_grub_glob;
+
+include 'pxelinux_config_common';
+
+prefix "/system/aii/nbp/pxelinux";
+# yeah, just pass non-glob for now
+#    get_repos is mocked with simple {} for now
+"kernel" = "http://abc.def/mykernel";
+"initrd" = "http://abc.def/path/to/initrd";

From b31044b4d6c3b6adaaee825c707af2a6c249db29 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 14 Apr 2021 14:05:17 +0200
Subject: [PATCH 19/28] pxelinux: support hostname lookup for EFI

---
 .../src/main/pan/quattor/aii/pxelinux/schema.pan       |  2 ++
 aii-pxelinux/src/main/perl/pxelinux.pm                 | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
index 9d8ccf42..0f43c56e 100644
--- a/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
+++ b/aii-pxelinux/src/main/pan/quattor/aii/pxelinux/schema.pan
@@ -19,6 +19,8 @@ type structure_pxelinux_pxe_info = {
       If this contains a '@pattern@' substring, the initrd path is generated based on
         the (first) enabled SPMA repository with name matching this glob pattern (without the '@').}
     "initrd" : string
+    @{try to resolve the hostname (when relevant) for EFI kernel and/or initrd; to use the ip instead of the hostname}
+    "efi_name_lookup" ? boolean
     "ksdevice"  : string with match(SELF, '^(bootif|link)$') || is_hwaddr(SELF) ||
         exists("/system/network/interfaces/" + escape(SELF))
     "kslocation" : type_absoluteURI
diff --git a/aii-pxelinux/src/main/perl/pxelinux.pm b/aii-pxelinux/src/main/perl/pxelinux.pm
index 74593acb..e825dca9 100755
--- a/aii-pxelinux/src/main/perl/pxelinux.pm
+++ b/aii-pxelinux/src/main/perl/pxelinux.pm
@@ -8,6 +8,7 @@ use File::stat;
 use File::Basename qw(dirname);
 use Time::localtime;
 use Readonly;
+use Socket;
 
 use parent qw (NCM::Component CAF::Path);
 
@@ -436,9 +437,6 @@ sub _kernel_initrd_path
 
     my $repos = get_repos($cfg);
 
-    use Data::Dumper;
-    $self->debug(1, "repos ".Dumper($repos));
-
     my $localhost_noglob = sub {
         my $txt = shift;
         $txt =~ s{\bLOCALHOST\b}{LOCALHOST}e;
@@ -472,6 +470,12 @@ sub _kernel_initrd_path
             };
         };
 
+        if ($pxe_config->{efi_name_lookup} &&
+            $res =~ m{^\((tftp|http(?:s)?),([^/:]+)(:\d+)?\)/(.*)$}) {
+            my $address = inet_ntoa(inet_aton($2));
+            $res = "($1,$address$3)/$4";
+        };
+
         push(@res, $res);
     }
 

From bd1d31434b2b108e369c84d827ec47d1555c12ac Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 1 Jul 2021 15:16:20 +0200
Subject: [PATCH 20/28] aii-core: support changing plugin modulename

---
 aii-core/src/main/perl/Shellfe.pm              |  2 +-
 .../src/test/perl/NCM/Component/doesexist.pm   | 11 +++++++++++
 aii-core/src/test/perl/shellfe.t               | 18 +++++++++++++++++-
 .../src/test/resources/modulename_exists.pan   |  3 +++
 .../test/resources/modulename_not_exists.pan   |  3 +++
 aii-ks/src/main/pan/quattor/aii/ks/schema.pan  |  1 +
 6 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 aii-core/src/test/perl/NCM/Component/doesexist.pm
 create mode 100644 aii-core/src/test/resources/modulename_exists.pan
 create mode 100644 aii-core/src/test/resources/modulename_not_exists.pan

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index 80f9f44f..3af3821b 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -529,7 +529,7 @@ sub run_plugin
 
     # This is here because CacheManager and Fetch objects may have
     # problems when they get out of scope.
-    my @modules = $only_modulename ? ($only_modulename) : sort keys %$tree;
+    my @modules = $only_modulename ? ($only_modulename) : map {$tree->{$_}->{plugin_modulename} || $_} sort keys %$tree;
 
     # Iterate over module names, handling each
     foreach my $modulename (@modules) {
diff --git a/aii-core/src/test/perl/NCM/Component/doesexist.pm b/aii-core/src/test/perl/NCM/Component/doesexist.pm
new file mode 100644
index 00000000..e5600e16
--- /dev/null
+++ b/aii-core/src/test/perl/NCM/Component/doesexist.pm
@@ -0,0 +1,11 @@
+package NCM::Component::doesexist;
+
+sub new {
+    my $class = shift;
+
+    return bless {}, $class;
+}
+
+sub Test {1};
+
+1;
diff --git a/aii-core/src/test/perl/shellfe.t b/aii-core/src/test/perl/shellfe.t
index b26a71f2..a2e0e260 100644
--- a/aii-core/src/test/perl/shellfe.t
+++ b/aii-core/src/test/perl/shellfe.t
@@ -1,9 +1,10 @@
 use strict;
 use warnings;
 use Test::More;
-use Test::Quattor qw(metaconfig);
+use Test::Quattor qw(metaconfig modulename_exists modulename_not_exists);
 use AII::Shellfe;
 use Cwd;
+use CAF::FileReader;
 
 use Readonly;
 use File::Basename qw(basename);
@@ -54,6 +55,21 @@ $cli->_metaconfig("somenode", {configuration => $cfg});
 my $fh = get_file(getcwd . "/target/test/cache/metaconfig/metaconfig/etc/something");
 is("$fh", "a=1\n\n", "metaconfig option rendered file in cache dir");
 
+# test modulename
+$cfg = get_config_for_profile('modulename_not_exists');
+
+$cli->{status} = 0;
+$cli->run_plugin({configuration => $cfg}, "/system/aii/osinstall", 'Test');
+is($cli->{status}, 16, "Failure");
+my $text;
+{local $/; open(my $fh, '<', $AII_LOG_FILE); $text = <$fh>;}
+like($text, qr{ERROR.*?Couldn't load plugin module doesnotexist},
+     "Failure due to osinstall module missing");
+
+$cli->{status} = 0;
+$cfg = get_config_for_profile('modulename_exists');
+$cli->run_plugin({configuration => $cfg}, "/system/aii/osinstall", 'Test');
+is($cli->{status}, 0, "No failure");
 
 
 done_testing;
diff --git a/aii-core/src/test/resources/modulename_exists.pan b/aii-core/src/test/resources/modulename_exists.pan
new file mode 100644
index 00000000..a682aefc
--- /dev/null
+++ b/aii-core/src/test/resources/modulename_exists.pan
@@ -0,0 +1,3 @@
+object template modulename_exists;
+
+"/system/aii/osinstall/doesnotexist/plugin_modulename" = "doesexist";
diff --git a/aii-core/src/test/resources/modulename_not_exists.pan b/aii-core/src/test/resources/modulename_not_exists.pan
new file mode 100644
index 00000000..3047b05f
--- /dev/null
+++ b/aii-core/src/test/resources/modulename_not_exists.pan
@@ -0,0 +1,3 @@
+object template modulename_not_exists;
+
+"/system/aii/osinstall/doesnotexist" = dict();
diff --git a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
index 4d4ba258..e51f6efc 100644
--- a/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
+++ b/aii-ks/src/main/pan/quattor/aii/ks/schema.pan
@@ -75,6 +75,7 @@ type structure_ks_mail = {
     for user customization are under /system/ks/hooks/.
 }
 type structure_ks_ks_info = {
+    "plugin_modulename" ? string
     "ackurl" ? type_absoluteURI with {deprecated(0, "ackurl is deprecated, use acklist instead"); true; }
     "acklist" ? type_absoluteURI[]
     "auth" : string[] = list ("enableshadow", "passalgo=sha512")

From 585f48ce5751509fe96c51c5074ca6384d1b8859 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 1 Jul 2021 15:33:34 +0200
Subject: [PATCH 21/28] aii-ks: add kickstart_post_script plugin module to
 generate the post section as a standalone script

---
 aii-ks/src/main/perl/ks.pm                    | 60 ++++++++++++----
 aii-ks/src/main/perl/ks_post_script.pm        | 70 +++++++++++++++++++
 aii-ks/src/test/perl/00-load.t                |  3 +-
 aii-ks/src/test/perl/kickstart_post_script.t  | 24 +++++++
 .../test/resources/kickstart_post_script.pan  |  3 +
 5 files changed, 144 insertions(+), 16 deletions(-)
 create mode 100644 aii-ks/src/main/perl/ks_post_script.pm
 create mode 100644 aii-ks/src/test/perl/kickstart_post_script.t
 create mode 100644 aii-ks/src/test/resources/kickstart_post_script.pan

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 668012bc..e8309f16 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -17,7 +17,7 @@ our $EC = LC::Exception::Context->new->will_store_all;
 
 our $this_app = $main::this_app;
 # Modules that may be interesting for hooks.
-our @EXPORT_OK = qw (ksuserhooks ksinstall_rpm get_repos replace_repo_glob);
+our @EXPORT_OK = qw (ksuserhooks ksinstall_rpm get_repos replace_repo_glob get_fqdn);
 
 # PAN paths for some of the information needed to generate the
 # Kickstart.
@@ -110,21 +110,36 @@ sub get_anaconda_version
     return $version;
 }
 
+sub _ks_filename
+{
+    my ($self, $ksdir, $fqdn) = @_;
+    return "$ksdir/$fqdn.ks";
+}
 
-# Opens the kickstart file and sets its handle as the default.
-sub ksopen
+sub ks_filename
 {
     my ($self, $cfg) = @_;
 
     my $fqdn = get_fqdn($cfg);
 
     my $ksdir = $this_app->option (KSDIROPT);
-    $self->debug(3,"Kickstart file directory = $ksdir");
+    $self->debug(3, "Kickstart file directory = $ksdir");
+
+    return $self->_ks_filename($ksdir, $fqdn);
+}
+
+
+# Opens the kickstart file and sets its handle as the default.
+sub ksopen
+{
+    my ($self, $cfg) = @_;
+
+    my $ks = CAF::FileWriter->open(
+        $self->ks_filename($cfg),
+        mode => 0664,
+        log => $this_app
+        );
 
-    my $ks = CAF::FileWriter->open ("$ksdir/$fqdn.ks",
-                                    mode => 0664,
-                                    log => $this_app
-                                    );
     select ($ks);
 }
 
@@ -1724,7 +1739,9 @@ EOF
 # this method.
 sub post_install_script
 {
-    my ($self, $config, $packages, $repos) = @_;
+    my ($self, $config, $packages, $repos, $is_kickstart) = @_;
+
+    $is_kickstart = 1 if ! defined($is_kickstart);
 
     my $tree = $config->getElement (KS)->getTree;
     my $version = get_anaconda_version($tree);
@@ -1734,21 +1751,28 @@ sub post_install_script
     my $logfile = '/tmp/post-log.log';
     my $logaction = log_action($config, $logfile);
 
-    print <<EOF;
+    if ($is_kickstart) {
+        print <<EOF;
 
 %post --nochroot
 
 test -f /tmp/pre-log.log && cp -a /tmp/pre-log.log /mnt/sysimage/root/
 EOF
 
-    ksuserhooks ($config, POSTNOCHROOTHOOK);
+        ksuserhooks ($config, POSTNOCHROOTHOOK);
 
-    print <<EOF;
+        print <<EOF;
 
 %end
 
 %post
 
+EOF
+
+    };
+
+    print <<EOF;
+
 # %post phase. The base system has already been installed. Let's do
 # some minor changes and prepare it for being configured.
 $logaction
@@ -1924,10 +1948,15 @@ echo 'End of post section'
 # Drain remote logger (0 if not relevant)
 sleep \$drainsleep
 
+EOF
+
+    if ($is_kickstart) {
+        print <<EOF;
+
 %end
 
 EOF
-
+    };
 }
 
 # Closes the Kickstart file and returns everything to its normal
@@ -1969,7 +1998,8 @@ sub Unconfigure
         return 1;
     }
 
-    my $ksdir = $main::this_app->option (KSDIROPT);
-    unlink ("$ksdir/$fqdn.ks");
+    unlink ($self->ks_filename($config));
     return 1;
 }
+
+1;
diff --git a/aii-ks/src/main/perl/ks_post_script.pm b/aii-ks/src/main/perl/ks_post_script.pm
new file mode 100644
index 00000000..6f5468b2
--- /dev/null
+++ b/aii-ks/src/main/perl/ks_post_script.pm
@@ -0,0 +1,70 @@
+#${PMpre} NCM::Component::ks_post_script${PMpost}
+
+# Generate a the ks post section of a node as a standalone script.
+
+use parent qw (NCM::Component::ks);
+
+use CAF::FileWriter;
+use NCM::Component::ks qw(get_fqdn);
+
+sub _ks_filename
+{
+    my ($self, $ksdir, $fqdn) = @_;
+    return "$ksdir/kickstart_post_$fqdn.sh";
+}
+
+# Cancels the open filewriter instance on the script
+#   and returns everything (the select magic/hack) to its normal state.
+# Returns the content of the cancelled filewriter instance
+sub ksclose
+{
+    my $fh = select;
+
+    my $text = "$fh";
+
+    select (STDOUT);
+
+    $fh->cancel();
+    $fh->close();
+
+    return "$text";
+}
+
+sub make_script
+{
+    my ($self, $cfg, $post_script) = @_;
+
+    my $fh = CAF::FileWriter->open($self->ks_filename($cfg), mode => 0755, log => $self);
+    print $fh $post_script;
+    $fh->close();
+}
+
+# Prints the kickstart file.
+sub Configure
+{
+    my ($self, $config) = @_;
+
+    my $fqdn = get_fqdn($config);
+    if ($CAF::Object::NoAction) {
+        $self->info ("Would run " . ref ($self) . " on $fqdn");
+        return 1;
+    }
+
+    $self->ksopen($config);
+
+    # ignore the packages that are treated in install for now
+    #   for now assume, all is there already
+    # this is not kickstart
+    #    won't generate the POSTNOCHROOTHOOK
+    #    no repos passed
+    $self->post_install_script ($config, [], {}, 0);
+
+    my $post_script = $self->ksclose();
+
+    $self->make_script($config, $post_script);
+
+    return 1;
+}
+
+
+1;
diff --git a/aii-ks/src/test/perl/00-load.t b/aii-ks/src/test/perl/00-load.t
index cfabbec9..d4dcfa29 100644
--- a/aii-ks/src/test/perl/00-load.t
+++ b/aii-ks/src/test/perl/00-load.t
@@ -3,6 +3,7 @@
 use strict;
 use warnings;
 use Test::Quattor;
-use Test::More tests => 1;
+use Test::More tests => 2;
 
 use_ok("NCM::Component::ks");
+use_ok("NCM::Component::ks_post_script");
diff --git a/aii-ks/src/test/perl/kickstart_post_script.t b/aii-ks/src/test/perl/kickstart_post_script.t
new file mode 100644
index 00000000..eec3e6c9
--- /dev/null
+++ b/aii-ks/src/test/perl/kickstart_post_script.t
@@ -0,0 +1,24 @@
+use strict;
+use warnings;
+use Test::More;
+use Test::Quattor qw(kickstart_post_script);
+use NCM::Component::ks_post_script;
+
+our $this_app = $main::this_app;
+
+$this_app->{CONFIG}->define('osinstalldir');
+$this_app->{CONFIG}->set('osinstalldir', '/some/path');
+
+my $obj = Test::Quattor::Object->new();
+
+my $ks = NCM::Component::ks_post_script->new('ks_post_script', $obj);
+
+my $cfg = get_config_for_profile('kickstart_post_script');
+
+$ks->Configure($cfg);
+
+my $fh = get_file('/some/path/kickstart_post_x.y.sh');
+like("$fh", qr{# %post phase}, "contains the post code");
+unlike("$fh", qr{^%post}m, "does not contain the %post tag");
+
+done_testing;
diff --git a/aii-ks/src/test/resources/kickstart_post_script.pan b/aii-ks/src/test/resources/kickstart_post_script.pan
new file mode 100644
index 00000000..2b347a7b
--- /dev/null
+++ b/aii-ks/src/test/resources/kickstart_post_script.pan
@@ -0,0 +1,3 @@
+object template kickstart_post_script;
+
+include 'kickstart';

From f5e772787134963aa939b581c627881c9a9dc7b6 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 5 Aug 2021 15:49:41 +0200
Subject: [PATCH 22/28] aii-ks: ks_post_script: generate the repos first

---
 aii-ks/src/main/perl/ks_post_script.pm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/aii-ks/src/main/perl/ks_post_script.pm b/aii-ks/src/main/perl/ks_post_script.pm
index 6f5468b2..1aaae4fd 100644
--- a/aii-ks/src/main/perl/ks_post_script.pm
+++ b/aii-ks/src/main/perl/ks_post_script.pm
@@ -51,13 +51,16 @@ sub Configure
     }
 
     $self->ksopen($config);
+    my ($packages, $repos) = $self->install ($config);
+    $self->ksclose();
 
+    $self->ksopen($config);
     # ignore the packages that are treated in install for now
     #   for now assume, all is there already
     # this is not kickstart
     #    won't generate the POSTNOCHROOTHOOK
     #    no repos passed
-    $self->post_install_script ($config, [], {}, 0);
+    $self->post_install_script ($config, [], $repos, 0);
 
     my $post_script = $self->ksclose();
 

From 056468bc006d15b854a256bfa329dbc5651031ca Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 22 Sep 2021 16:56:55 +0200
Subject: [PATCH 23/28] aii-ks: for kickstart as script, run ks-post-install in
 "post"     section (and without extra reboot)

---
 aii-ks/src/main/perl/ks.pm | 80 ++++++++++++++++++++++++++++----------
 1 file changed, 59 insertions(+), 21 deletions(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index e8309f16..77c0b04b 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -148,9 +148,11 @@ sub ksopen
 # only the post_reboot script.
 sub kspostreboot_hereopen
 {
+    my ($self, $dest) = @_;
+
     print <<EOF;
 
-cat <<End_Of_Post_Reboot > /etc/rc.d/init.d/ks-post-reboot
+cat <<End_Of_Post_Reboot > $dest
 EOF
 }
 
@@ -1254,7 +1256,7 @@ EOF
 
 sub kspostreboot_tail
 {
-    my $config = shift;
+    my ($config, $as_service) = @_;
 
     ksuserscript ($config, POSTREBOOTSCRIPT);
 
@@ -1262,6 +1264,11 @@ sub kspostreboot_tail
 
 success
 
+EOF
+
+    if ($as_service) {
+        print <<EOF;
+
 if [ -x /usr/bin/systemctl ]; then
     rm -f /etc/systemd/system/system-update.target.wants/ks-post-reboot.service
     rmdir --ignore-fail-on-non-empty /etc/systemd/system/system-update.target.wants
@@ -1271,11 +1278,22 @@ else
     rm -f /etc/rc.d/rc3.d/S86ks-post-reboot
 fi
 
+EOF
+    }
+
+    print <<EOF;
+
 echo 'End of ks-post-reboot'
 
 # Drain remote logger (0 if not relevant)
 sleep \\\$drainsleep
 
+EOF
+
+    if ($as_service) {
+        # don't trigger the reboot if this is not run as a service
+        print <<EOF;
+
 if [ -x /usr/bin/systemctl ]; then
     /usr/bin/systemctl --no-wall reboot
 else
@@ -1283,19 +1301,20 @@ else
 fi
 
 EOF
+    }
 }
 
 
 # Prints the post_reboot script.
 sub post_reboot_script
 {
-    my ($self, $config) = @_;
+    my ($self, $config, $kspi_filename, $kspi_as_service) = @_;
 
-    kspostreboot_header ($config);
+    kspostreboot_header ($config, $kspi_filename);
     ksuserhooks ($config, POSTREBOOTHOOK);
     ksquattor_config ($config);
     ksuserhooks ($config, POSTREBOOTENDHOOK);
-    kspostreboot_tail ($config);
+    kspostreboot_tail ($config, $kspi_as_service);
 }
 
 # The way Anaconda handles the bootloader and MBR can screw software
@@ -1743,6 +1762,11 @@ sub post_install_script
 
     $is_kickstart = 1 if ! defined($is_kickstart);
 
+    # Location where to put the ks-post-install script
+    my $kspi_filename = $is_kickstart ? "/etc/rc.d/init.d/ks-post-reboot" : "/root/ks-post-reboot-script";
+    # Run ks-post-install script as service/unit or not
+    my $kspi_as_service = $is_kickstart;
+
     my $tree = $config->getElement (KS)->getTree;
     my $version = get_anaconda_version($tree);
 
@@ -1785,8 +1809,8 @@ set -x
 
 EOF
 
-    $self->kspostreboot_hereopen;
-    $self->post_reboot_script ($config);
+    $self->kspostreboot_hereopen($kspi_filename);
+    $self->post_reboot_script ($config, $kspi_filename);
     $self->kspostreboot_hereclose;
     ksuserhooks ($config, POSTHOOK);
 
@@ -1805,7 +1829,7 @@ EOF
     }
 
     # restore UEFI pxeboot first
-    if ($tree->{pxeboot}) {
+    if ($is_kickstart && $tree->{pxeboot}) {
         print <<EOF;
 #
 # restore pxeboot as first boot order in case efibootmgr is around
@@ -1861,18 +1885,23 @@ EOF
         };
     };
 
-    # Systemd unit file
-    #   Targets
-    #     basic.target is default dependency
-    #     we need network.target
-    #       and ks-post-reboot needs to be started after it (not in parallel)
-    #     syslog/rsyslog and sshd are a nice-to-have
-    #   Type oneshot disables timeout and is blocking
-    #     other services/targets can be started in parallel
+        print <<EOF;
 
-    print <<EOF;
+chmod 640 $kspi_filename
 
-chmod +x /etc/rc.d/init.d/ks-post-reboot
+EOF
+
+    if ($kspi_as_service) {
+        # Systemd unit file
+        #   Targets
+        #     basic.target is default dependency
+        #     we need network.target
+        #       and ks-post-reboot needs to be started after it (not in parallel)
+        #     syslog/rsyslog and sshd are a nice-to-have
+        #   Type oneshot disables timeout and is blocking
+        #     other services/targets can be started in parallel
+
+        print <<EOF;
 
 if [ -x /usr/bin/systemctl ]; then
     cat <<EOF_reboot_unit > /usr/lib/systemd/system/ks-post-reboot.service
@@ -1895,7 +1924,7 @@ Wants=network.service
 
 [Service]
 Type=oneshot
-ExecStart=/etc/rc.d/init.d/ks-post-reboot start
+ExecStart=$kspi_filename start
 ExecStop=/bin/true
 ExecStopPost=/usr/bin/rm -fv /system-update
 FailureAction=reboot
@@ -1909,7 +1938,7 @@ TTYVHangup=yes
 EOF_reboot_unit
 
     # /system-update is expected to be a symlink, identifying which update script to run
-    ln -sf /etc/rc.d/init.d/ks-post-reboot /system-update
+    ln -sf $kspi_filename /system-update
 
     # The documentation recommends creating the .wants symlink directly instead of using [Install] and 'systemctl enable'
     mkdir -p /etc/systemd/system/system-update.target.wants
@@ -1923,10 +1952,19 @@ LogLevel=debug
 EOF_systemd_logging
 
 else
-    ln -s /etc/rc.d/init.d/ks-post-reboot /etc/rc.d/rc3.d/S86ks-post-reboot
+    ln -s $kspi_filename /etc/rc.d/rc3.d/S86ks-post-reboot
 fi
 
 EOF
+    } else {
+        # Run the script
+        print <<EOF;
+
+$kspi_filename start
+
+EOF
+
+    }
 
     my @acklist;
     if ($config->elementExists (ACKLIST) ) {

From 749d9563bbfeeb8cb400c0a79784262f444261e2 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 24 Sep 2021 14:13:12 +0200
Subject: [PATCH 24/28] aii-ks: fix permission of ks-post-reboot script

---
 aii-ks/src/main/perl/ks.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aii-ks/src/main/perl/ks.pm b/aii-ks/src/main/perl/ks.pm
index 77c0b04b..3394e773 100755
--- a/aii-ks/src/main/perl/ks.pm
+++ b/aii-ks/src/main/perl/ks.pm
@@ -1887,7 +1887,7 @@ EOF
 
         print <<EOF;
 
-chmod 640 $kspi_filename
+chmod 550 $kspi_filename
 
 EOF
 

From fca85b0fbf41ead1654e59baec37af9598920bfc Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 20 Oct 2021 13:12:11 +0200
Subject: [PATCH 25/28] aii-core: add ansible command

---
 aii-core/src/main/perl/Shellfe.pm | 36 ++++++++++++++++++++++++++++++-
 aii-core/src/test/perl/shellfe.t  |  7 ++++++
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index 3af3821b..2a9bf882 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -60,7 +60,7 @@ use constant LOCKFILE   => '/var/lock/quattor/aii';
 use constant RETRIES    => 6;
 use constant TIMEOUT    => 60;
 use constant PARTERR_ST => 16;
-use constant COMMANDS   => qw (remove configure install boot rescue firmware livecd status metaconfig);
+use constant COMMANDS   => qw (remove configure install boot rescue firmware livecd status metaconfig ansible);
 use constant INCLUDE    => 'include';
 use constant CAFILE     => 'ca_file';
 use constant CADIR      => 'ca_dir';
@@ -192,6 +192,12 @@ sub app_options
                     '(can be a regexp)',
          DEFAULT => undef },
 
+       { NAME    => 'ansible=s',
+         HELP    => 'Node(s) to generate all ansible playbooks for, ' .
+                    'relative to the cachemanager cachepath for that host' .
+                    '(can be a regexp)',
+         DEFAULT => undef },
+
        { NAME    => CDBURL.'=s',
          HELP    => 'URL for CDB location',
          DEFAULT => undef },
@@ -578,6 +584,14 @@ sub run_plugin
                 $plug->set_active_config($st->{configuration});
             }
 
+            if ($method == 'ansible_command') {
+                # make role for component name, and also pass it via configuration hack
+                # AII code does not support aliased components
+                my $role = $configuration->{ansible}->{playbook}->add_role(component name);
+                # placeholder for current/last active role
+                $configuration->{ansible}->{role} = $role;
+            }
+
             # The plugin method has to return success
             my $res = eval { $plug->$method ($st->{configuration}) };
             if ($@) {
@@ -1122,6 +1136,26 @@ sub _metaconfig
     $self->run_plugin($st, '/software/components/metaconfig', 'aii_command', 'metaconfig');
 }
 
+=item _metaconfig
+
+Runs the ansible_command method of all components of the node.
+
+=cut
+
+sub _ansible
+{
+    my ($self, $node, $st) = @_;
+    my $playbook = AII::Playbook->new($node, log => $self);
+
+    # ugly hack: pass it to the component via the configuration instance
+    $st->{configuration}->{ansible}->{playbook} = $playbook;
+
+    $self->run_plugin($st, '/software/components', 'ansible_command');
+
+    # for now, write it in the cache dir
+    $playbook->write($st->{configuration}->{cache_path}. "/ansible");
+}
+
 =item get_cache_time
 
 Return the mtime of the C<node> AII statefile.
diff --git a/aii-core/src/test/perl/shellfe.t b/aii-core/src/test/perl/shellfe.t
index a2e0e260..aac8e5e6 100644
--- a/aii-core/src/test/perl/shellfe.t
+++ b/aii-core/src/test/perl/shellfe.t
@@ -55,6 +55,13 @@ $cli->_metaconfig("somenode", {configuration => $cfg});
 my $fh = get_file(getcwd . "/target/test/cache/metaconfig/metaconfig/etc/something");
 is("$fh", "a=1\n\n", "metaconfig option rendered file in cache dir");
 
+# Test ansible
+$cfg = get_config_for_profile('metaconfig');
+$cli->_ansible("somenode", {configuration => $cfg});
+
+$fh = get_file(getcwd . "/target/test/cache/metaconfig/ansible/metaconfig.yml");
+is("$fh", "abc", "metaconfig playbook rendered in cache dir");
+
 # test modulename
 $cfg = get_config_for_profile('modulename_not_exists');
 

From 90c968fd46935c8548d165b7a6904f79f0a6834f Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Wed, 20 Oct 2021 13:12:30 +0200
Subject: [PATCH 26/28] aii-core: add ansible playbook, role and task module

---
 aii-core/src/main/perl/Playbook.pm | 83 ++++++++++++++++++++++++++++++
 aii-core/src/main/perl/Role.pm     | 46 +++++++++++++++++
 aii-core/src/main/perl/Task.pm     | 35 +++++++++++++
 aii-core/src/test/perl/playbook.t  | 46 +++++++++++++++++
 4 files changed, 210 insertions(+)
 create mode 100644 aii-core/src/main/perl/Playbook.pm
 create mode 100644 aii-core/src/main/perl/Role.pm
 create mode 100644 aii-core/src/main/perl/Task.pm
 create mode 100644 aii-core/src/test/perl/playbook.t

diff --git a/aii-core/src/main/perl/Playbook.pm b/aii-core/src/main/perl/Playbook.pm
new file mode 100644
index 00000000..4f775303
--- /dev/null
+++ b/aii-core/src/main/perl/Playbook.pm
@@ -0,0 +1,83 @@
+#${PMpre} AII::Playbook${PMpost}
+
+use LC::Exception qw (SUCCESS);
+use parent qw(CAF::Object);
+use CAF::TextRender;
+use CAF::Path;
+
+use AII::Role;
+
+# hosts: playbook hosts
+sub _initialize
+{
+    my ($self, $hosts, %opts) = @_;
+
+    %opts = () if !%opts;
+
+    $self->{log} = $opts{log} if $opts{log};
+
+    $self->{data} = {
+        hosts => $hosts
+    };
+    $self->{roles} = [];
+
+    return SUCCESS;
+}
+
+sub add_role
+{
+    my ($self, $name) = @_;
+    my $role = AII::Role->new($name, log => $self);
+    push @{$self->{roles}}, $role;
+    return $role;
+}
+
+
+# Generate hashref to render into yaml
+sub make_data {
+    my $self = shift;
+
+    # make copy of basic data
+    my $data = {%{$self->{data}}};
+
+    # add roles
+    $data->{roles} = [map {$_->{name}} @{$self->{roles}}];
+
+    return $data;
+}
+
+# Generate playbook and roles
+# root: base working dir
+sub write
+{
+    my ($self, $root) = @_;
+
+    # Make roles subdir in root
+    my $cafpath = CAF::Path::mkcafpath(log => $self);
+    $cafpath->directory("$root/roles");
+
+    # Generate all roles and playbook data
+    my $files = {
+        main => $self->make_data()
+    };
+
+    foreach my $role (@{$self->{roles}}) {
+        $files->{"roles/$role->{name}"} = $role->make_data();
+    }
+
+    # Write
+    foreach my $filename (sort keys %$files) {
+        my $trd = CAF::TextRender->new(
+            'yamlmulti',
+            {'host' => [$files->{$filename}]},  # use yamlmulti to bypass arrayref issue
+            log => $self,
+            );
+        my $fh = $trd->filewriter(
+            "$root/$filename.yml",
+            log => $self,
+            );
+        $fh->close();
+    };
+}
+
+1;
diff --git a/aii-core/src/main/perl/Role.pm b/aii-core/src/main/perl/Role.pm
new file mode 100644
index 00000000..4a0fa8e2
--- /dev/null
+++ b/aii-core/src/main/perl/Role.pm
@@ -0,0 +1,46 @@
+#${PMpre} AII::Role${PMpost}
+
+use LC::Exception qw (SUCCESS);
+use parent qw(CAF::Object);
+
+use AII::Task;
+
+# name: name of role
+sub _initialize
+{
+    my ($self, $name, %opts) = @_;
+
+    %opts = () if !%opts;
+
+    $self->{log} = $opts{log} if $opts{log};
+
+    $self->{name} = $name;
+    $self->{data} = {
+    };
+    $self->{tasks} = [];
+
+    return SUCCESS;
+}
+
+sub add_task
+{
+    my ($self, $name) = @_;
+    my $task = AII::Task->new($name, log => $self);
+    push @{$self->{tasks}}, $task;
+    return $task;
+}
+
+# Generate hashref to render into yaml
+sub make_data {
+    my $self = shift;
+
+    # make copy of basic data
+    my $data = {%{$self->{data}}};
+
+    # add tasks
+    $data->{tasks} = [map {$_->make_data()} @{$self->{tasks}}];
+
+    return $data;
+}
+
+1;
diff --git a/aii-core/src/main/perl/Task.pm b/aii-core/src/main/perl/Task.pm
new file mode 100644
index 00000000..18d6e780
--- /dev/null
+++ b/aii-core/src/main/perl/Task.pm
@@ -0,0 +1,35 @@
+#${PMpre} AII::Task${PMpost}
+
+use LC::Exception qw (SUCCESS);
+use parent qw(CAF::Object);
+
+# name: name of task
+sub _initialize
+{
+    my ($self, $name, %opts) = @_;
+
+    %opts = () if !%opts;
+
+    $self->{log} = $opts{log} if $opts{log};
+
+    $self->{name} = $name;
+    $self->{data} = {
+    };
+
+    return SUCCESS;
+}
+
+# Generate hashref to render into yaml
+sub make_data {
+    my $self = shift;
+
+    # make copy of basic data
+    my $data = {%{$self->{data}}};
+
+    # add tasks
+    $data->{name} = $self->{name};
+
+    return $data;
+}
+
+1;
diff --git a/aii-core/src/test/perl/playbook.t b/aii-core/src/test/perl/playbook.t
new file mode 100644
index 00000000..e79f3bf1
--- /dev/null
+++ b/aii-core/src/test/perl/playbook.t
@@ -0,0 +1,46 @@
+use strict;
+use warnings;
+use Test::More;
+use Test::Quattor;
+use Test::Quattor::Object;
+use Cwd;
+
+use AII::Playbook;
+
+$CAF::Object::NoAction = 1;
+
+my $obj = Test::Quattor::Object->new();
+
+my $pb = AII::Playbook->new("myhost", log => $obj);
+
+my $root = getcwd . "/target/test/playbook/myhost";
+
+$pb->write($root);
+
+# No roles
+my $fh = get_file("$root/main.yml");
+is("$fh", "---\n- hosts: myhost\n  roles: []\n");
+
+# Add role
+my $role = $pb->add_role("first");
+isa_ok ($role, "AII::Role", "Correct class after add_role");
+my $task = $role->add_task("task1");
+isa_ok ($task, "AII::Task", "Correct class after add_task");
+
+$task = $role->add_task("task2");
+
+$role = $pb->add_role("second");
+$task = $role->add_task("task3");
+
+$pb->write($root);
+
+# No roles
+$fh = get_file("$root/main.yml");
+is("$fh", "---\n- hosts: myhost\n  roles:\n  - first\n  - second\n");
+$fh = get_file("$root/roles/first.yml");
+is("$fh", "---\n- tasks:\n  - name: task1\n  - name: task2\n");
+$fh = get_file("$root/roles/second.yml");
+is("$fh", "---\n- tasks:\n  - name: task3\n");
+
+
+done_testing;

From 539923d2073c11cbff40cb8d13af862dceca2752 Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Thu, 21 Oct 2021 06:51:57 +0200
Subject: [PATCH 27/28] aii-core: add more ansible tests

---
 aii-core/src/main/perl/Shellfe.pm             | 32 ++++++++++++-------
 .../src/test/perl/NCM/Component/ansible.pm    | 15 +++++++++
 aii-core/src/test/perl/shellfe.t              | 17 ++++++----
 aii-core/src/test/resources/ansible.pan       |  8 +++++
 4 files changed, 54 insertions(+), 18 deletions(-)
 create mode 100644 aii-core/src/test/perl/NCM/Component/ansible.pm
 create mode 100644 aii-core/src/test/resources/ansible.pan

diff --git a/aii-core/src/main/perl/Shellfe.pm b/aii-core/src/main/perl/Shellfe.pm
index 2a9bf882..6362f01e 100755
--- a/aii-core/src/main/perl/Shellfe.pm
+++ b/aii-core/src/main/perl/Shellfe.pm
@@ -36,6 +36,7 @@ use File::Basename qw(basename dirname);
 use DB_File;
 use Readonly;
 use Parallel::ForkManager 0.7.6;
+use AII::Playbook;
 
 use NCM::Component::metaconfig 18.6.0;
 
@@ -535,12 +536,19 @@ sub run_plugin
 
     # This is here because CacheManager and Fetch objects may have
     # problems when they get out of scope.
-    my @modules = $only_modulename ? ($only_modulename) : map {$tree->{$_}->{plugin_modulename} || $_} sort keys %$tree;
+    my %pmodules;
+    if ($only_modulename) {
+        $pmodules{$only_modulename} = $only_modulename;
+    } else {
+        %pmodules = map {$_ => $tree->{$_}->{plugin_modulename} || $tree->{$_}->{'ncm-module'} || $_} keys %$tree;
+    }
 
     # Iterate over module names, handling each
-    foreach my $modulename (@modules) {
+    #    TODO: when dealing with ansible, this order should be resolved via the dependencies
+    foreach my $pname (sort keys %pmodules) {
+        my $modulename = $pmodules{$pname};
         if ($modulename !~ m/^[a-zA-Z_]\w+(::[a-zA-Z_]\w+)*$/) {
-            $self->error ("Invalid Perl identifier $modulename specified as a plug-in. Skipping.");
+            $self->error ("Invalid Perl identifier $modulename specified as a plug-in for $pname. Skipping.");
             $self->{status} = PARTERR_ST;
             next;
         }
@@ -553,7 +561,7 @@ sub run_plugin
             $self->debug (4, "Loading plugin module $modulename");
             eval (USEMODULE . $modulename);
             if ($@) {
-                $self->error ("Couldn't load plugin module $modulename for path $path: $@");
+                $self->error ("Couldn't load plugin module $modulename for $pname for path $path: $@");
                 $self->{status} = PARTERR_ST;
                 next;
             }
@@ -579,19 +587,19 @@ sub run_plugin
                 $self->plugin_handler($modulename, @_);
             });
 
+            if ($method eq 'ansible_command') {
+                my $ansible = $st->{configuration}->{ansible};
+                # make role for component name, and also pass it via configuration hack
+                my $role = $ansible->{playbook}->add_role($pname);
+                # placeholder for current/last active role
+                $ansible->{role} = $role;
+            }
+
             # Set active config
             if ($plug->can('set_active_config')) {
                 $plug->set_active_config($st->{configuration});
             }
 
-            if ($method == 'ansible_command') {
-                # make role for component name, and also pass it via configuration hack
-                # AII code does not support aliased components
-                my $role = $configuration->{ansible}->{playbook}->add_role(component name);
-                # placeholder for current/last active role
-                $configuration->{ansible}->{role} = $role;
-            }
-
             # The plugin method has to return success
             my $res = eval { $plug->$method ($st->{configuration}) };
             if ($@) {
diff --git a/aii-core/src/test/perl/NCM/Component/ansible.pm b/aii-core/src/test/perl/NCM/Component/ansible.pm
new file mode 100644
index 00000000..bffc94a8
--- /dev/null
+++ b/aii-core/src/test/perl/NCM/Component/ansible.pm
@@ -0,0 +1,15 @@
+package NCM::Component::ansible;
+
+sub new {
+    my $class = shift;
+
+    return bless {}, $class;
+}
+
+sub ansible_command {
+    my ($self, $configuration) = @_;
+    $configuration->{ansible}->{role}->add_task("mytask");
+    return 1;  # must return success
+}
+
+1;
diff --git a/aii-core/src/test/perl/shellfe.t b/aii-core/src/test/perl/shellfe.t
index aac8e5e6..62b126f6 100644
--- a/aii-core/src/test/perl/shellfe.t
+++ b/aii-core/src/test/perl/shellfe.t
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use Test::More;
-use Test::Quattor qw(metaconfig modulename_exists modulename_not_exists);
+use Test::Quattor qw(metaconfig modulename_exists modulename_not_exists ansible);
 use AII::Shellfe;
 use Cwd;
 use CAF::FileReader;
@@ -50,17 +50,22 @@ is_deeply($cli->_download_options('ccm'), {}, "empty config returns hashref for
 
 # Test metaconfig
 my $cfg = get_config_for_profile('metaconfig');
-$cli->_metaconfig("somenode", {configuration => $cfg});
+$cli->_metaconfig("somenode", {configuration => $cfg, name => 'somename'});
 
 my $fh = get_file(getcwd . "/target/test/cache/metaconfig/metaconfig/etc/something");
 is("$fh", "a=1\n\n", "metaconfig option rendered file in cache dir");
 
 # Test ansible
-$cfg = get_config_for_profile('metaconfig');
-$cli->_ansible("somenode", {configuration => $cfg});
+$cfg = get_config_for_profile('ansible');
+$cli->_ansible("ansinode", {configuration => $cfg, name => 'ansiname'});
+
+$fh = get_file(getcwd . "/target/test/cache/ansible/ansible/main.yml");
+is("$fh", "---\n- hosts: ansinode\n  roles:\n  - ansible\n  - myalias\n", "ansible playbook rendered in cache dir");
+$fh = get_file(getcwd . "/target/test/cache/ansible/ansible/roles/ansible.yml");
+is("$fh", "---\n- tasks:\n  - name: mytask\n", "ansible role1 rendered in cache dir");
+$fh = get_file(getcwd . "/target/test/cache/ansible/ansible/roles/myalias.yml");
+is("$fh", "---\n- tasks:\n  - name: mytask\n", "ansible role2 rendered in cache dir");
 
-$fh = get_file(getcwd . "/target/test/cache/metaconfig/ansible/metaconfig.yml");
-is("$fh", "abc", "metaconfig playbook rendered in cache dir");
 
 # test modulename
 $cfg = get_config_for_profile('modulename_not_exists');
diff --git a/aii-core/src/test/resources/ansible.pan b/aii-core/src/test/resources/ansible.pan
new file mode 100644
index 00000000..f5dd5490
--- /dev/null
+++ b/aii-core/src/test/resources/ansible.pan
@@ -0,0 +1,8 @@
+object template ansible;
+
+prefix "/software/components/myalias";
+"ncm-module" = "ansible";
+"data" = 1;
+
+prefix "/software/components/ansible";
+"data" = 2;

From ace5b1ba1b479a7f4aeb0362f061071d38a2f0fd Mon Sep 17 00:00:00 2001
From: stdweird <stijn.deweirdt@ugent.be>
Date: Fri, 22 Oct 2021 11:11:29 +0200
Subject: [PATCH 28/28] aii-core: ansible task: support passing data on task
 creation

---
 aii-core/src/main/perl/Role.pm    | 4 ++--
 aii-core/src/main/perl/Task.pm    | 5 ++---
 aii-core/src/test/perl/playbook.t | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/aii-core/src/main/perl/Role.pm b/aii-core/src/main/perl/Role.pm
index 4a0fa8e2..46d96650 100644
--- a/aii-core/src/main/perl/Role.pm
+++ b/aii-core/src/main/perl/Role.pm
@@ -24,8 +24,8 @@ sub _initialize
 
 sub add_task
 {
-    my ($self, $name) = @_;
-    my $task = AII::Task->new($name, log => $self);
+    my ($self, $name, $data) = @_;
+    my $task = AII::Task->new($name, $data, log => $self);
     push @{$self->{tasks}}, $task;
     return $task;
 }
diff --git a/aii-core/src/main/perl/Task.pm b/aii-core/src/main/perl/Task.pm
index 18d6e780..c4700954 100644
--- a/aii-core/src/main/perl/Task.pm
+++ b/aii-core/src/main/perl/Task.pm
@@ -6,15 +6,14 @@ use parent qw(CAF::Object);
 # name: name of task
 sub _initialize
 {
-    my ($self, $name, %opts) = @_;
+    my ($self, $name, $data, %opts) = @_;
 
     %opts = () if !%opts;
 
     $self->{log} = $opts{log} if $opts{log};
 
     $self->{name} = $name;
-    $self->{data} = {
-    };
+    $self->{data} = $data || {};
 
     return SUCCESS;
 }
diff --git a/aii-core/src/test/perl/playbook.t b/aii-core/src/test/perl/playbook.t
index e79f3bf1..fca74360 100644
--- a/aii-core/src/test/perl/playbook.t
+++ b/aii-core/src/test/perl/playbook.t
@@ -30,7 +30,7 @@ isa_ok ($task, "AII::Task", "Correct class after add_task");
 $task = $role->add_task("task2");
 
 $role = $pb->add_role("second");
-$task = $role->add_task("task3");
+$task = $role->add_task("task3", {some => 'thing'});
 
 $pb->write($root);
 
@@ -40,7 +40,7 @@ is("$fh", "---\n- hosts: myhost\n  roles:\n  - first\n  - second\n");
 $fh = get_file("$root/roles/first.yml");
 is("$fh", "---\n- tasks:\n  - name: task1\n  - name: task2\n");
 $fh = get_file("$root/roles/second.yml");
-is("$fh", "---\n- tasks:\n  - name: task3\n");
+is("$fh", "---\n- tasks:\n  - name: task3\n    some: thing\n");
 
 
 done_testing;