pubRunMap

#!/usr/bin/env python

# first load the standard libraries from python
# we require at least python 2.7
#from sys import *
from __future__ import print_function
import sys
if sys.version_info[0]==2 and not sys.version_info[1]>=5:
    print("Sorry, this program requires at least python 2.5")
    print("You can download a more current python version from python.org and compile it")
    print("into your homedir (or anywhere) with 'configure --prefix ~/python27'; make;")
    print("then run this program by specifying your own python executable like this: ")
    print("   ~/python27/bin/python <scriptFile>")
    print("or add ~/python27/bin to your PATH before /usr/bin")
    exit(1)

# load default python packages
import logging, optparse, os, tarfile, tempfile, copy, shutil, glob
from os.path import *

# add <scriptDir>/lib/ to package search path
progFile = os.path.abspath(sys.argv[0])
progDir  = os.path.dirname(progFile)
pubToolsLibDir = os.path.join(progDir, "lib")
sys.path.insert(0, pubToolsLibDir)

# now load our own libraries
import maxRun, pubStore, pubConf, pubGeneric, pubAlg
from maxCommon import *

ANNOTDIGITS=5 # number of digits to use for annotation ID 

# === COMMAND LINE INTERFACE, OPTIONS AND HELP ===
parser = optparse.OptionParser("""usage: %prog [options] <algorithmName> <in> <out> - run an algorithm on a directory of fulltext files and write results to out directory

If <in> and <out> are directories:
    search <in> for fulltext files
    submit cluster jobs to process these to <out>
If <in> and <out> are files:
    parse article text from in
    write results to out
""")

parser.add_option("-d", "--debug", dest="debug", action="store_true", help="show debug messages") 
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show more debug messages") 
parser.add_option("-r", "--reduce", dest="reduceFile", action="store", metavar="OUTFILE", help="do not only submit the jobs, but shove them through parasol, continue until all jobs are done, then run reducer and write to output file")
parser.add_option("-t", "--test", dest="test", action="store_true", help="do not run on cluster")
(options, args) = parser.parse_args()

# ==== MAIN =======
def main(args, options):
    if args==[]:
        parser.print_help()
        exit(1)

    pubGeneric.setupLogging(progFile, options)

    algName, inName, outName = args[:3]
    paramList = args[3:]
    paramDict = pubGeneric.stringListToDict(paramList)

    alg = pubAlg.getAlg(algName)

    if not "map" in args:
        mustBeEmptyDir(outName, makeDir=True)
        if options.test:
            runner=None
        else:
            runner = pubGeneric.makeClusterRunner(__file__, algName=basename(inName)+"-map")
        pubAlg.findFilesSubmitJobs(algName, "map", inName, outName, ".marshal.gz", \
            paramDict, runNow=options.reduceFile, runner=runner)
        if options.reduceFile:
            outFilename = options.reduceFile
            pubAlg.runReduce(algName, outName, outFilename)
            
    else:
        #mustExist(inName)
        reader = pubStore.PubReaderFile(inName)
        pubAlg.runMap(reader, alg, outName)
        reader.close()

main(args, options)