Merge pull request #239 from HubbleStack/develop

Merge to master (prep for 2016.9.0)

README.rst

@@ -79,7 +79,8 @@ it to the minions.
     cd hubblestack-nova.git
     mkdir -p /srv/salt/_modules/
     cp _modules/hubble.py /srv/salt/_modules/
-    cp -a hubblestack_nova /srv/salt/
+    cp -a hubblestack_nova_profiles /srv/salt/
+    cp -a hubblestack_nova_modules /srv/salt/
 
     salt \* saltutil.sync_modules
     salt \* hubble.sync
@@ -93,15 +94,15 @@ Usage
 
 There are four primary functions in the hubble.py module:
 
-1. ``hubble.sync`` will sync the ``hubblestack_nova/`` directory to the minion(s).
+1. ``hubble.sync`` will sync the ``hubblestack_nova_profiles/`` and ``hubblestack_nova_modules/`` directories to the minion(s).
 2. ``hubble.load`` will load the synced audit modules and their yaml configuration files.
 3. ``hubble.audit`` will audit the minion(s) using the YAML profile(s) you provide as comma-separated arguments
 4. ``hubble.top`` will audit the minion(s) using the ``top.nova`` configuration.
 
 ``hubble.audit`` takes two optional arguments. The first is a comma-separated
 list of paths.  These paths can be files or directories within the
-``hubblestack_nova`` directory. The second argument allows for toggling Nova
-configuration, such as verbosity, level of detail, etc.
+``hubblestack_nova_profiles`` directory. The second argument allows for
+toggling Nova configuration, such as verbosity, level of detail, etc.
 
 If ``hubble.audit`` is run without targeting any audit configs or directories,
 it will instead run ``hubble.top`` with no arguments.
@@ -119,9 +120,9 @@ Here are some example calls:
     # Run hubble.top with the default topfile (top.nova)
     salt \* hubble.top
 
-    # Run all yaml configs and tags under salt://hubblestack_nova/foo/ and
-    # salt://hubblestack_nova/bar, but only run audits with tags starting
-    # with "CIS"
+    # Run all yaml configs and tags under salt://hubblestack_nova_profiles/foo/
+    # and salt://hubblestack_nova_profiles/bar, but only run audits with tags
+    # starting with "CIS"
     salt \* hubble.audit foo,bar tags='CIS*'
 
 .. _nova_usage_topfile:
@@ -222,6 +223,7 @@ In order to run the audits once daily, you can use the following schedule:
           show_profile: True
         returner: splunk_nova_return
         return_job: False
+        run_on_start: False
 
 .. _nova_configuration:
 
@@ -241,7 +243,8 @@ configurable via pillar. The defaults are shown below:
     hubblestack:
       nova:
         saltenv: base
-        dir: salt://hubblestack_nova
+        module_dir: salt://hubblestack_nova_modules
+        profile_dir: salt://hubblestack_nova_profiles
 
 2. By default, ``hubble.audit`` will call ``hubble.load`` (which in turn calls
 ``hubble.sync``) in order to ensure that it is auditing with the most up-to-date

_modules/hubble.py

@@ -10,7 +10,8 @@
 See README for documentation
 
 Configuration:
-    - hubblestack:nova:dir
+    - hubblestack:nova:module_dir
+    - hubblestack:nova:profile_dir
     - hubblestack:nova:saltenv
     - hubblestack:nova:autoload
     - hubblestack:nova:autosync
@@ -299,7 +300,8 @@ def top(topfile='top.nova',
     Arguments:
 
     topfile
-        The path of the topfile, relative to your hubblestack_nova directory.
+        The path of the topfile, relative to your hubblestack_nova_profiles
+        directory.
 
     verbose
         Whether to show additional information about audits, including
@@ -413,21 +415,27 @@ def top(topfile='top.nova',
     return results
 
 
-def sync():
+def sync(clean=False):
     '''
-    Sync the nova audit modules from the saltstack fileserver.
+    Sync the nova audit modules and profiles from the saltstack fileserver.
 
     The modules should be stored in the salt fileserver. By default nova will
-    search the base environment for a top level ``hubblestack_nova`` directory,
-    unless otherwise specified via pillar or minion config
-    (``hubblestack:nova:dir``)
+    search the base environment for a top level ``hubblestack_nova_modules``
+    directory, unless otherwise specified via pillar or minion config
+    (``hubblestack:nova:module_dir``)
 
-    Modules will just be cached in the normal minion cachedir
+    The profiles should be stored in the salt fileserver. By default nova will
+    search the base environment for a top level ``hubblestack_nova_profiles``
+    directory, unless otherwise specified via pillar or minion config
+    (``hubblestack:nova:profile_dir``)
 
-    Returns the minion's path to the cached directory
+    Modules and profiles will be cached in the normal minion cachedir
 
-    NOTE: This function will also clean out existing files at the cached
-    location, as cp.cache_dir doesn't clean out old files
+    Returns a boolean representing success
+
+    NOTE: This function will optionally clean out existing files at the cached
+    location, as cp.cache_dir doesn't clean out old files. Pass ``clean=True``
+    to enable this behavior
 
     CLI Examples:
 
@@ -437,35 +445,44 @@ def sync():
         salt '*' nova.sync saltenv=hubble
     '''
     log.debug('syncing nova modules')
-    nova_dir = __salt__['config.get']('hubblestack:nova:dir', 'salt://hubblestack_nova')
+    nova_profile_dir = __salt__['config.get']('hubblestack:nova:profile_dir',
+                                              'salt://hubblestack_nova_profiles')
+    nova_module_dir = __salt__['config.get']('hubblestack:nova:module_dir',
+                                             'salt://hubblestack_nova_modules')
     saltenv = __salt__['config.get']('hubblestack:nova:saltenv', 'base')
 
-    # Support optional salt:// in config
-    if 'salt://' in nova_dir:
-        path = nova_dir
-        _, _, nova_dir = nova_dir.partition('salt://')
-    else:
-        path = 'salt://{0}'.format(nova_dir)
-
     # Clean previously synced files
-    __salt__['file.remove'](_hubble_dir())
-    # Sync the files
-    cached = __salt__['cp.cache_dir'](path, saltenv=saltenv)
-
-    if cached and isinstance(cached, list):
-        # Success! Trim the paths
-        cachedir = _hubble_dir()
-        ret = [relative.partition(cachedir)[2] for relative in cached]
-        return ret
-    else:
-        if isinstance(cached, list):
-            # Nothing was found
-            return cached
+    if clean:
+        for nova_dir in _hubble_dir():
+            __salt__['file.remove'](nova_dir)
+
+    synced = []
+    for i, nova_dir in enumerate((nova_module_dir, nova_profile_dir)):
+        # Support optional salt:// in config
+        if 'salt://' in nova_dir:
+            path = nova_dir
+            _, _, nova_dir = nova_dir.partition('salt://')
+        else:
+            path = 'salt://{0}'.format(nova_dir)
+
+        # Sync the files
+        cached = __salt__['cp.cache_dir'](path, saltenv=saltenv)
+
+        if cached and isinstance(cached, list):
+            # Success! Trim the paths
+            cachedir = os.path.dirname(_hubble_dir()[i])
+            ret = [relative.partition(cachedir)[2] for relative in cached]
+            synced.extend(ret)
         else:
-            # Something went wrong, there's likely a stacktrace in the output
-            # of cache_dir
-            raise CommandExecutionError('An error occurred while syncing: {0}'
-                                        .format(cached))
+            if isinstance(cached, list):
+                # Nothing was found
+                synced.extend(cached)
+            else:
+                # Something went wrong, there's likely a stacktrace in the output
+                # of cache_dir
+                raise CommandExecutionError('An error occurred while syncing: {0}'
+                                            .format(cached))
+    return synced
 
 
 def load():
@@ -474,8 +491,10 @@ def load():
     '''
     if __salt__['config.get']('hubblestack:nova:autosync', True):
         sync()
-    if not os.path.isdir(_hubble_dir()):
-        return False, 'No synced nova modules found'
+
+    for nova_dir in _hubble_dir():
+        if not os.path.isdir(nova_dir):
+            return False, 'No synced nova modules/profiles found'
 
     log.debug('loading nova modules')
 
@@ -491,18 +510,28 @@ def load():
 
 def _hubble_dir():
     '''
-    Generate the local minion directory to which nova modules are synced
+    Generate the local minion directories to which nova modules and profiles
+    are synced
+
+    Returns a tuple of two paths, the first for nova modules, the second for
+    nova profiles
     '''
-    nova_dir = __salt__['config.get']('hubblestack:nova:dir', 'hubblestack_nova')
+    nova_profile_dir = __salt__['config.get']('hubblestack:nova:profile_dir',
+                                              'salt://hubblestack_nova_profiles')
+    nova_module_dir = __salt__['config.get']('hubblestack:nova:module_dir',
+                                             'salt://hubblestack_nova_modules')
+    dirs = []
     # Support optional salt:// in config
-    if 'salt://' in nova_dir:
-        _, _, nova_dir = nova_dir.partition('salt://')
-    saltenv = __salt__['config.get']('hubblestack:nova:saltenv', 'base')
-    cachedir = os.path.join(__opts__.get('cachedir'),
-                            'files',
-                            saltenv,
-                            nova_dir)
-    return cachedir
+    for nova_dir in (nova_module_dir, nova_profile_dir):
+        if 'salt://' in nova_dir:
+            _, _, nova_dir = nova_dir.partition('salt://')
+        saltenv = __salt__['config.get']('hubblestack:nova:saltenv', 'base')
+        cachedir = os.path.join(__opts__.get('cachedir'),
+                                'files',
+                                saltenv,
+                                nova_dir)
+        dirs.append(cachedir)
+    return tuple(dirs)
 
 
 def _calculate_compliance(results):
@@ -526,7 +555,7 @@ def _get_top_data(topfile):
     '''
     Helper method to retrieve and parse the nova topfile
     '''
-    topfile = os.path.join(_hubble_dir(), topfile)
+    topfile = os.path.join(_hubble_dir()[1], topfile)
 
     try:
         with open(topfile) as handle:
@@ -558,7 +587,7 @@ class NovaLazyLoader(LazyLoader):
     '''
 
     def __init__(self):
-        super(NovaLazyLoader, self).__init__([_hubble_dir()],
+        super(NovaLazyLoader, self).__init__(_hubble_dir(),
                                              opts=__opts__,
                                              tag='nova')
         self.__data__ = {}
@@ -597,6 +626,14 @@ def refresh_file_mapping(self):
                         # Nova only supports .py and .yaml
                         if ext not in ['.py', '.yaml']:
                             continue
+                        # Python only in the modules directory, yaml only
+                        # in the profiles directory. This is hacky but was a
+                        # quick fix.
+                        nova_module_cache, nova_profile_cache = _hubble_dir()
+                        if ext == '.py' and fpath.startswith(nova_profile_cache):
+                            continue
+                        if ext == '.yaml' and fpath.startswith(nova_module_cache):
+                            continue
                         if f_withext in self.disabled:
                             #log.trace(
                             #    'Skipping {0}, it is disabled by configuration'.format(

hubblestack_nova/modules/command.py → hubblestack_nova_modules/command.py

@@ -3,6 +3,11 @@
 Hubble Nova plugin for running arbitrary commands and checking the output of
 those commands
 
+This module is deprecated, and must be explicitly enabled in pillar/minion
+config via the hubblestack:nova:enable_command_module (should be set to True
+to enable this module). This allows nova to run arbitrary commands via yaml
+profiles.
+
 :maintainer: HubbleStack / basepi
 :maturity: 2016.7.0
 :platform: All
@@ -101,6 +106,14 @@ def audit(data_list, tags, verbose=False, show_profile=False, debug=False):
         log.debug(__tags__)
 
     ret = {'Success': [], 'Failure': [], 'Controlled': []}
+
+    if __tags__ and not __salt__['config.get']('hubblestack:nova:enable_command_module',
+                                               False):
+        ret['Error'] = ['command module has not been explicitly enabled in '
+                        'config. Please set hubblestack:nova:enable_command_module '
+                        'to True in pillar or minion config to allow this module.']
+        return ret
+
     for tag in __tags__:
         if fnmatch.fnmatch(tag, tags):
             for tag_data in __tags__[tag]:

hubblestack_nova/modules/grep.py → hubblestack_nova_modules/grep.py

@@ -113,9 +113,9 @@ def audit(data_list, tags, verbose=False, show_profile=False, debug=False):
                 if isinstance(grep_args, str):
                     grep_args = [grep_args]
 
-                grep_ret = __salt__['file.grep'](name,
-                                                 tag_data['pattern'],
-                                                 *grep_args).get('stdout')
+                grep_ret = _grep(name,
+                                 tag_data['pattern'],
+                                 *grep_args).get('stdout')
 
                 found = False
                 if grep_ret:
@@ -278,3 +278,61 @@ def _get_tags(data):
                         formatted_data.pop('data')
                         ret[tag].append(formatted_data)
     return ret
+
+
+def _grep(path,
+          pattern,
+          *opts):
+    '''
+    Grep for a string in the specified file
+
+    .. note::
+        This function's return value is slated for refinement in future
+        versions of Salt
+
+    path
+        Path to the file to be searched
+
+        .. note::
+            Globbing is supported (i.e. ``/var/log/foo/*.log``, but if globbing
+            is being used then the path should be quoted to keep the shell from
+            attempting to expand the glob expression.
+
+    pattern
+        Pattern to match. For example: ``test``, or ``a[0-5]``
+
+    opts
+        Additional command-line flags to pass to the grep command. For example:
+        ``-v``, or ``-i -B2``
+
+        .. note::
+            The options should come after a double-dash (as shown in the
+            examples below) to keep Salt's own argument parser from
+            interpreting them.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' file.grep /etc/passwd nobody
+        salt '*' file.grep /etc/sysconfig/network-scripts/ifcfg-eth0 ipaddr -- -i
+        salt '*' file.grep /etc/sysconfig/network-scripts/ifcfg-eth0 ipaddr -- -i -B2
+        salt '*' file.grep "/etc/sysconfig/network-scripts/*" ipaddr -- -i -l
+    '''
+    path = os.path.expanduser(path)
+
+    split_opts = []
+    for opt in opts:
+        try:
+            opt = salt.utils.shlex_split(opt)
+        except AttributeError:
+            opt = salt.utils.shlex_split(str(opt))
+        split_opts.extend(opt)
+
+    cmd = ['grep'] + split_opts + [pattern, path]
+    try:
+        ret = __salt__['cmd.run_all'](cmd, python_shell=False, ignore_retcode=True)
+    except (IOError, OSError) as exc:
+        raise CommandExecutionError(exc.strerror)
+
+    return ret