Pass command line arguments as dictionary instead of structure.

Author: rplzzz

README.md

@@ -52,6 +52,8 @@ to add the `--user` flag.
 
 ## Running
 
+### Standalone runs
+
 To run your models under Cassandra you will first need to prepare a
 configuration file.  Configuration files are written in the INI
 format.  In this format, the file is divided into sections, the names
@@ -78,3 +80,38 @@ Depending on how your cluster is set up, you might have to include
 additional flags for `mpirun`, such as a hostfile giving the names of
 the hosts you will be running on.  An example job script for systems
 using the Slurm resource manager is included in `extras/mptest.zsh`.
+
+### Running from another python program
+
+It isn't strictly necessary to run `cassandra_main.py` as a standalone
+script.  You could instead run from another python program, or from a
+Jupyter notebook.  Besides giving access to the interactive features
+of notebooks, running this way could allow you to import components
+(see "Adding New Models" below) that are stored in modules outside the
+Cassandra package.  To do this, you will need to import
+`cassandra_main` as a module.  If you want to add a new component, you
+will also need the `add_new_component` function from
+`cassandra.compfactory`.
+
+Once you've imported the necessary modules, continue by adding your
+custom components, if any.  Then, you will need to create the
+structure used as the argument to the `main()` function.  In the
+standalone version, this structure is created by the `argparse` module
+from the command line arguments.  
+
+```python
+from cassandra.cassandra_main import main
+## If including a component from an external module, add these lines
+## too.  (Make sure mycomp.py is in your python path!)
+from cassandra.compfactory import add_new_component
+import mycomp.py
+
+```
+
+
+
+## Adding New Models
+
+Cassandra's interface to models is provided by objects called
+_components_.  To add a model to the system, you have to create a
+component to run the model.  Components 

cassandra/cassandra_main.py

@@ -18,12 +18,11 @@
 import os
 
 
-def bootstrap_sp(argvals):
+def bootstrap_sp(args):
     """
-    Bootstrap the multithreaded (single processing) version of the
-    calculation.
+    Bootstrap the multithreaded (single processing) version of the calculation.
 
-    :param argvals: Command line arguments parsed by argparse.
+    :param args: Dictionary of command line arguments parsed by argparse.
     :return: (component-list, capability-table)
     """
 
@@ -32,17 +31,17 @@ def bootstrap_sp(argvals):
     from cassandra.compfactory import create_component
 
     # Configure logger
-    if argvals.logdir is None:
-        logging.basicConfig(stream=sys.stdout, level=argvals.loglvl)
+    if args['logdir'] is None:
+        logging.basicConfig(stream=sys.stdout, level=args['loglvl'])
         logging.info(f'This is Cassandra version {__version__}.')
     else:
-        os.makedirs(argvals.logdir, exist_ok=True)
-        logging.basicConfig(filename=f'{argvals.logdir}/cassandra.log',
-                            level=argvals.loglvl, filemode='w')
+        os.makedirs(args['logdir'], exist_ok=True)
+        logging.basicConfig(filename=f"{args['logdir']}/cassandra.log",
+                            level=args['loglvl'], filemode='w')
         # Write to screen the location of the logging output
-        print(f'This is Cassandra version {__version__}.  Output will be logged to {argvals.logdir}/cassandra.log')
+        print(f"This is Cassandra version {__version__}.  Output will be logged to {args['logdir']}/cassandra.log")
 
-    cfgfile_name = argvals.ctlfile
+    cfgfile_name = args['ctlfile']
 
     # initialize the structures that will receive the data we are
     # parsing from the file
@@ -68,24 +67,51 @@ def bootstrap_sp(argvals):
 # end of bootstrap_sp
 
 
-def main(argvals):
+def main(args):
+    """
+    Cassandra main entry function.
+
+    :param args: Dictionary of command line arguments parsed by argparse.
+
+    This function starts up the Cassandra framework, selecting either the
+    single-processing or multi-processing mode, as required.  It's set up to be
+    run from the script block at the end of the module, but it could be called
+    from another program if desired.  To do that you will need to set up a
+    dictionary simulating the command-line parameters from the stand-alone
+    version.  The parameters that need to be supplied are:
+       ctlfile : Name of the configuration ("control") file.
+       mp      : Flag indicating whether we are running in MP mode
+       logdir  : Directory for log files. In SP mode this can be None, in 
+                 which case log outputs go to stdout
+       verbose : Flag indicating whether to produce debugging output.
+       quiet   : Flag indicating whether to suppress output except for warnings
+                 and error messages
+    
+    Keep in mind that this function will throw an exception if any of the
+    components fail (whether by exception or by returning a failure code).  It's
+    up to whatever code is calling it to handle any cleanup associated with the
+    failure.  This is especially important if you are running in MP mode, as
+    failing to do this cleanup can hang the entire calculation.
+
+    """
+
 
     # Set the appropriate logging level
     # NB: You MUST NOT call any logging functions until either bootstrap_mp
     #     or bootstrap_sp has been called.
-    if argvals.verbose:
-        argvals.loglvl = logging.DEBUG
-    elif argvals.quiet:
-        argvals.loglvl = logging.WARNING
+    if args['verbose']:
+        args['loglvl'] = logging.DEBUG
+    elif args['quiet']:
+        args['loglvl'] = logging.WARNING
     else:
-        argvals.loglvl = logging.INFO
+        args['loglvl'] = logging.INFO
 
-    if argvals.mp:
+    if args['mp']:
         # See notes in mp.py about side effects of importing that module.
         from cassandra.mp import bootstrap_mp, finalize
-        (component_list, cap_table) = bootstrap_mp(argvals)
+        (component_list, cap_table) = bootstrap_mp(args)
     else:
-        (component_list, cap_table) = bootstrap_sp(argvals)
+        (component_list, cap_table) = bootstrap_sp(args)
 
     # We will look up "general" in the cap_table and process any
     # global parameters here, but in the current version we don't
@@ -99,7 +125,7 @@ def main(argvals):
 
     # Wait for all component threads to complete before printing end
     # message.
-    if argvals.mp:
+    if args['mp']:
         # The RAB thread will always be first in the thread list
         component_threads = threads[1:]
     else:
@@ -114,7 +140,7 @@ def main(argvals):
     # if the RAB is present, it is always the first in the list.
     nfail = 0
 
-    if argvals.mp:
+    if args['mp']:
         reg_comps = component_list[1:]
         rab_comp = component_list[0]
     else:
@@ -140,7 +166,7 @@ def main(argvals):
 
     # If this is a multiprocessing calculation, then we need to
     # perform the finalization procedure
-    if argvals.mp:
+    if args['mp']:
         finalize(component_list[0], threads[0])
 
     logging.info("\nFIN.")
@@ -163,7 +189,7 @@ def main(argvals):
 
     try:
         # status is the number of component failures.
-        status = main(argvals)
+        status = main(vars(argvals))
     except Exception as err:
         if argvals.mp:
             from mpi4py import MPI

cassandra/compfactory.py

@@ -36,3 +36,24 @@ def create_component(compname, cap_tbl):
         raise RuntimeError(f'Unknown component type {compname}')
 
     return _available_components[compname](cap_tbl)
+
+def add_new_component(compname, classobj):
+    """Add a new type of component to the list of available components.
+
+    :param compname: Name by which the new component will be known in 
+                     configuration files.
+    :param classobj: The class object for the class implementing the 
+                     component.  The class MUST be a subclass of ComponentBase.
+
+    Calling this function will add the requested component to the available
+    components table, allowing it to be parsed from config files and instantiated
+    by the system.  The class passed in as class object must be a subclass of 
+    ComponentBase.  The class inheritance isn't checked, so passing some random class
+    might appear to work, or it might work in actuality, but nothing is guaranteed.
+
+    """
+
+    global _available_components
+    
+    _available_components[compname] = classobj
+    

cassandra/mp.py

@@ -36,38 +36,41 @@
 import os
 
 
-def bootstrap_mp(argvals):
+def bootstrap_mp(args):
     """Bootstrap the multiprocessing system.
 
+    :param args: Dictionary of arguments parsed from the command line.
+
     1. Parse configuration files
     2. Distribute configuration information to peers
     3. Create RABs
     4. Create components
     5. Add remote capabilities to capability table
-    6. Return component list (including RAB).  The main loop will be able to
-    start these components in the usual way.
+    6. Return component list (including the RAB first in the list).  The main loop 
+       will be able to start these components in the usual way.
+
     """
 
     world = MPI.COMM_WORLD
     rank = world.Get_rank()
 
-    if argvals.logdir is None:
+    if args['logdir'] is None:
         logdir = 'logs'
     else:
-        logdir = argvals.logdir
+        logdir = args['logdir']
 
     os.makedirs(logdir, exist_ok=True)
 
-    logging.basicConfig(filename=f'{logdir}/cassandra-{rank}.log', level=argvals.loglvl,
+    logging.basicConfig(filename=f'{logdir}/cassandra-{rank}.log', level=args['loglvl'],
                         filemode='w')
 
     if rank == SUPERVISOR_RANK:
         # Print the location of the logs so that it will appear in the
         # output file of batch jobs.
         print(f'\nThis is Cassandra version {__version__}.  Log output will be written to {logdir}.')
-        my_assignment = distribute_assignments_supervisor(argvals)
+        my_assignment = distribute_assignments_supervisor(args)
     else:
-        my_assignment = distribute_assignments_worker(argvals)
+        my_assignment = distribute_assignments_worker(args)
 
     # my_assignment will be a dictionary of configuration sections assigned to
     # this process.  We need to create and initialize the components assigned to
@@ -104,10 +107,10 @@ def bootstrap_mp(argvals):
     return (comps, cap_tbl)
 
 
-def distribute_assignments_worker(argvals):
+def distribute_assignments_worker(args):
     """Prepare to receive component assignments
 
-    :param argvals: Arguments structure parsed by argparse.  Not currently used,
+    :param args: Dictionary of arguments parsed by argparse.  Not currently used,
                     but included in case we eventually want to have command line
                     arguments that affect the way assignments are handled.
     :return: Dictionary of component definitions for components assigned to this
@@ -125,10 +128,10 @@ def distribute_assignments_worker(argvals):
     return assignments
 
 
-def distribute_assignments_supervisor(argvals):
+def distribute_assignments_supervisor(args):
     """Parse config file and distribute component assignments
 
-    :param argvals: Arguments structure parsed by argparse.
+    :param args: Dictionary of arguments parsed by argparse.
     :return: Dictionary of component definitions for components assigned to this
                     process (i.e., the supervisor).
 
@@ -150,7 +153,7 @@ def distribute_assignments_supervisor(argvals):
 
     from configobj import ConfigObj
 
-    config = ConfigObj(argvals.ctlfile)
+    config = ConfigObj(args['ctlfile'])
 
     # Get list of section names
     section_names = list(config.keys())