pubRunMapReduce

#!/usr/bin/env python

# first load the standard libraries from python
# we require at least python 2.7
#from sys import *
from __future__ import print_function
import sys
if sys.version_info[0]==2 and not sys.version_info[1]>=5:
    print("Sorry, this program requires at least python 2.5")
    print("You can download a more current python version from python.org and compile it")
    print("into your homedir (or anywhere) with 'configure --prefix ~/python27'; make;")
    print("then run this program by specifying your own python executable like this: ")
    print("   ~/python27/bin/python <scriptFile>")
    print("or add ~/python27/bin to your PATH before /usr/bin")
    exit(1)

# load default python packages
import logging, optparse
from os.path import *

# add <scriptDir>/lib/ to package search path
progFile = abspath(sys.argv[0])
progDir  = dirname(progFile)
pubToolsLibDir = join(progDir, "lib")
sys.path.insert(0, pubToolsLibDir)

# now load our own libraries
import pubAlg, pubGeneric, pubConf

# === COMMAND LINE INTERFACE, OPTIONS AND HELP ===
parser = optparse.OptionParser("""usage: %prog [options] <algorithmName> <textDir> <outputFilename> - run a map-reduce algorithm on a directory of text files
""")

parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show debug messages") 
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show more debug messages") 
parser.add_option("-r", "--onlyReduce", dest="onlyReduce", action="store_true", help="only run reducer") 
parser.add_option("-t", "--test", dest="test", action="store_true", help="only run on a single file") 
(options, args) = parser.parse_args()

# ==== MAIN =======
def main(args, options):
    if args==[]:
        parser.print_help()
        exit(1)

    pubGeneric.setupLogging(progFile, options)

    algName, textDirSpec, outFilename = args[:3]
    paramList = args[3:]
    paramDict = pubGeneric.stringListToDict(paramList)

    textDirs = pubConf.resolveTextDirs(textDirSpec)
    runner = pubGeneric.makeClusterRunner(__file__)
    pubAlg.mapReduce(algName, textDirs, paramDict, outFilename, skipMap=options.onlyReduce, \
        onlyTest=options.test, runner=runner)

main(args, options)