Update_HCPCS_DEVV5.gradle

/*
****************************
* HCPCS Vocabulary update  *
* Version 0.0.7            *
****************************
*/

group 'com.sci.vocabulary'
version '1.0-SNAPSHOT'

apply plugin: 'java'

sourceCompatibility = 1.8

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
    }
    dependencies {
        classpath 'org.apache.poi:poi:3.9'
        classpath 'org.apache.poi:poi-ooxml:3.9'
        classpath group: 'commons-io', name: 'commons-io', version: '2.5'
    }
}

/**
 * JAVA-CODE SECTION.
 *
 * Temporary java code are located here. If future it's need to move it toi separate java-module.
 */

import groovy.io.FileType
import org.apache.poi.ss.usermodel.Cell
import org.apache.poi.ss.usermodel.Row
import org.apache.poi.ss.usermodel.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory
import org.apache.poi.ss.usermodel.Sheet
import org.apache.commons.io.FileUtils
import java.nio.charset.StandardCharsets

/*
task compileOne(type: JavaCompile) {
    source = sourceSets.main.java.srcDir
    include 'inc.odysseus.utils/StringBuilderWrapper.java'
    classpath = sourceSets.main.output.classesDir
    destinationDir = sourceSets.main.output.classesDir
}
compileOne.options.compilerArgs = ["-sourcepath", "${projectDir}/src/main/java"]
*/

/**
 * Auxiliary classes. TODO: Further should be used java-defined and compiled only
 */
public class StringBuilderWrapper implements java.lang.Appendable {
    private static final String IDENT_CHAR = " " *4
    private StringBuilder wrapped

    public StringBuilderWrapper() {
        wrapped = new StringBuilder()
    }
    public StringBuilderWrapper(String value) {
        this()
        wrapped.append(IDENT_CHAR).append(value)
    }

    public static StringBuilderWrapper getBuilder() {
        return new StringBuilderWrapper()
    }

    public static String readFromFile(File file) {
        if (file == null)
            return ''
        def builder = getBuilder()
        def br = new BufferedReader(new FileReader(file))
        def strLine
        while ((strLine = br.readLine()) != null) {
            builder.append('\n').append(strLine)
        }
        return builder.toString()
    }

    @Override
    Appendable append(CharSequence csq) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq)
        return this
    }
    @Override
    Appendable append(CharSequence csq, int start, int end) throws IOException {
        wrapped.append(IDENT_CHAR).append(csq, start, end)
        return this
    }
    @Override
    Appendable append(char c) throws IOException {
        wrapped.append(IDENT_CHAR).append(c)
        return this
    }
    @Override
    String toString() {
        return wrapped.toString()
    }
}

class StringHelper {
    public static final String CSV_FIELD_SEPARATOR = ";"

    static String normalizeString(String value) {
        String result = new String(value);
        // If empty - just return
        if (StringUtils.isBlank(value))
            return result;
        boolean doubleQuotesProcessed = false
        // Replace all number of double quotes by twice double quotes
        if (result.indexOf("\"") >= 0) {
            result = result.replaceAll("(\"{1,1})", "\"\"");
            doubleQuotesProcessed = true;
        }
        // Replace all double quotes by
        if (result.indexOf(CSV_FIELD_SEPARATOR) >= 0 || doubleQuotesProcessed) {
            result = String.format("\"%s\"", result);
        }
        // Replace all carriage return to space
        if (result.indexOf("\n") >= 0) {
            result = result.replaceAll("\n", " ");
        }
        // Replace all carriage return to space
        if (result.indexOf("\r") >= 0) {
            result = result.replaceAll("\r", " ");
        }
        // Replace the 'EN DASH' (U+2013) by the 'HYPHEN-MINUS' (U+002D)
        if (result.indexOf("\u2013") >= 0) {
            result = result.replaceAll("\u2013", "\u002D");
        }
        return result;
    }
}

/**
 * JAVA-CODE SECTION
 */

// Load HCPCS properties
Properties props = new Properties()
props.load(new FileInputStream("${project.rootDir}/Update_HCPCS.properties"))
props.each {prop ->
    project.ext.set(prop.key, prop.value)
}
props.load(new FileInputStream("${project.getBuildFile().toString().replace('.gradle', '.properties')}"))
props.each {prop ->
    project.ext.set(prop.key, prop.value)
}

def tempDir = File.createTempDir()
tempDir.deleteOnExit()

/**
 * Prepare 'Manual Table' to load data with 'Manual Mapping'
 */
task prepareManualTable(description: project.properties['prepareManualTable.description']) << {
    println '\n\n*****************************************************'
    println '*                                                   *'
    println "*     '${description}' action started...      *"
    println '*                                                   *'
    println '*****************************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['prepareManualTable.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['prepareManualTable.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['prepareManualTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['prepareManualTable.streamErrFileName']}"

    def db = [un       : project.properties['db.userName'],
              pw       : project.properties['db.password'],
              tns      : project.properties['db.tnsName'],
              sqlfile  : sqlFile,
              tableName: "${project.properties['manualTableName']}",
              logfile  : logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus -L ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile} ${db.tableName}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    // Get Manual Table from editors
    println new StringBuilderWrapper('\'Manual Table\' copy started...')
    def srcFile = "${project.properties['basePath']}/${project.properties['prepareManualTable.manualWorkResultPath']}/${project.properties['manualTableFileName']}"
    def dest = "${project.properties['basePath']}"
    def result = copy {
        from srcFile
        into dest
    }
    if (result.getDidWork()) {
        println new StringBuilderWrapper('file copied')
    } else {
        println new StringBuilderWrapper('file isn\'t copied')
    }
    println new StringBuilderWrapper('\'Manual Table\' copy done...')

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Import data of 'Manual Mapping' into 'Manual Table'
 */
task importManualTable (dependsOn: prepareManualTable, description: project.properties['importManualTable.description']) << {
    println '\n\n*****************************************************'
    println '*                                                   *'
    println "*      '${description}' action started...      *"
    println '*                                                   *'
    println '*****************************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def dataFile = "${project.properties['basePath']}/${project.properties['manualTableFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['importManualTable.logFileName']}"
    def discardFile = "${project.properties['logPath']}/${project.properties['importManualTable.discardFileName']}"
    def badFile = "${project.properties['logPath']}/${project.properties['importManualTable.badFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['importManualTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['importManualTable.streamErrFileName']}"

    def bosOut = new ByteArrayOutputStream(), bosErr = new ByteArrayOutputStream()
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    exec {
        executable "sqlldr"
        args "USERID=${project.properties['db.userName']}/${project.properties['db.password']}@${project.properties['db.tnsName']} " +
                "CONTROL=${project.properties['basePath']}/${project.properties['importManualTable.controlFileName']} " +
                "DATA=${dataFile} " +
                "LOG=${logFile} " +
                "DISCARD=${discardFile} " +
                "BAD=${badFile}"
        standardOutput = bosOut
        errorOutput = bosErr
    }
    def bisOut = new ByteArrayInputStream(bosOut.toByteArray())
    bisOut.withReader { out->
        out.eachLine {
            sOut.append(it).append('\n')
        }
    }
    def bisErr = new ByteArrayInputStream(bosErr.toByteArray())
    bisErr.withReader { out ->
        out.eachLine {
            sErr.append(it).append('\n')
        }
    }

    /**
     *  Write to console.
     *  When import prefer to write stream log to console (it's shorter).
     */
    System.out.println(sOut.toString())

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"

}

/**
 * Apply data from 'Manual Mapping' to be ready for further processing in 'Load Stage'
 */
task applyManualTable(dependsOn: importManualTable, description: project.properties['applyManualTable.description']) << {
    println '\n\n*****************************************************'
    println '*                                                   *'
    println "*      '${description}' action started...       *"
    println '*                                                   *'
    println '*****************************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['applyManualTable.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['applyManualTable.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['applyManualTable.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['applyManualTable.streamErrFileName']}"

    def db = [un       : project.properties['db.userName'],
              pw       : project.properties['db.password'],
              tns      : project.properties['db.tnsName'],
              sqlfile  : sqlFile,
              tableName: "${project.properties['manualTableName']}",
              logfile  : logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile} ${db.tableName}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will run 'Load stage' script.
 * Seems it's is specific for each vocabulary.
 */
task loadStage(dependsOn: applyManualTable, description: project.properties['loadStage.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*     '${description}' action started...    *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['loadStage.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['loadStage.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['loadStage.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['loadStage.streamErrFileName']}"
    def logFileProcedureDrug = "${project.properties['logPath']}/${project.properties['loadStage.logProcedureDrugFileName']}"
    def logFileMapDrugVocabulary = "${project.properties['logPath']}/${project.properties['loadStage.logMapDrugVocabularyFileName']}"

    def db = [un                      : project.properties['db.userName'],
              pw                      : project.properties['db.password'],
              tns                     : project.properties['db.tnsName'],
              sqlfile                 : sqlFile,
              logfile                 : logFile,
              contextPath             : "${project.properties['basePath']}",
              logfileProcedureDrug    : logFileProcedureDrug,
              logfileMapDrugVocabulary: logFileMapDrugVocabulary]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile} ${db.contextPath} ${db.logfileProcedureDrug} ${db.logfileMapDrugVocabulary}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))
    lf = new File(logFileProcedureDrug)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))
    lf = new File(logFileMapDrugVocabulary)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will run 'Genetic update' script.
 * It's common for all updated vocabularies and located in '\Vocabulary-v5.0\working\generic_update.sql' file.
 */
task genericUpdate(dependsOn: loadStage, description: project.properties['genericUpdate.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*     '${description}' action started...      *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['genericUpdate.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['genericUpdate.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['genericUpdate.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['genericUpdate.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to logfile
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * This task will run scripts which allows to see the vocabulary update results
 */
task checkUpdateResults(dependsOn: genericUpdate, description: project.properties['checkUpdateResults.description']) << {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*  '${description}' action started...   *"
    println '*                                             *'
    println '***********************************************'

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    def sqlFile = "${project.properties['basePath']}/${project.properties['checkUpdateResults.sqlFileName']}"
    def logFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.logFileName']}"
    def streamLogFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.streamLogFileName']}"
    def streamErrorFile = "${project.properties['logPath']}/${project.properties['checkUpdateResults.streamErrFileName']}"

    def db = [un     : project.properties['db.userName'],
              pw     : project.properties['db.password'],
              tns    : project.properties['db.tnsName'],
              sqlfile: sqlFile,
              logfile: logFile]
    def sOut = new StringBuilderWrapper(), sErr = new StringBuilderWrapper()
    def sqlPlus = """sqlplus ${db.un}/${db.pw}@${db.tns} @${db.sqlfile} ${db.logfile}""".execute()
    sqlPlus.consumeProcessOutput(sOut, sErr)
    sqlPlus.waitFor()

    // Finish action checkpoint
    def timeFinish = System.currentTimeMillis()

    /**
     * Write to console.
     * Write compact SQLPlus's spool log
     */
    File lf = new File(logFile)
    if (lf.exists())
        System.out.println(StringBuilderWrapper.readFromFile(lf))

    // Write to log file
    File f = new File(streamLogFile)
    f.write(sOut.toString())

    // Write to error file
    f = new File(streamErrorFile)
    f.write(sErr.toString())

    /* Check for errors */
    if (Boolean.valueOf(project.properties['stop.on.error'].toString()) && (sqlPlus.exitValue() != 0 || (sErr != null && sErr.toString().length() > 0))) {
        throw new GradleException(String.format(project.properties['error.message.with.code'], sqlPlus.exitValue(), name))
    }

    // Measure time execution
    def timeElapsed = (timeFinish - timeStart) / 1000
    println "Time elapsed: ${timeElapsed} seconds."

    println "*** '${description}' action done ***"
}

/**
 * Bootstrap action
 *
 * TODO: Needs to find solution to measure whole task chain execution.
 * TODO: This one via doFirst/doLast actions isn't working.
 */
task startUpdate(dependsOn: checkUpdateResults, description: project.properties['startUpdate.description']) {
    println '\n\n***********************************************'
    println '*                                             *'
    println "*      '${description}' action started...       *"
    println '*                                             *'
    println '***********************************************'

    println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema...\n"

    project.ext['logPath'] = "${project.properties['basePath']}/${project.properties['baseLogPath']}"
    def folder = new File(project.properties['logPath'])
    if (!folder.exists()) {
        folder.mkdirs()
        println "Log directory is created: ${folder.getPath()}"
    } else {
        FileUtils.cleanDirectory(folder)
        println "Log directory is cleaned: ${folder.getPath()}"
    }

    // Start action checkpoint
    def timeStart = System.currentTimeMillis()

    doLast {
        // Finish action checkpoint
        def timeFinish = System.currentTimeMillis()

        // Measure time execution
        def timeElapsed = (timeFinish - timeStart) / 1000
        println "\n\nTotal time elapsed: ${timeElapsed} seconds."
        println "Updating of '${project.properties['vocabulary.display.name']}' vocabulary on the '${project.properties['db.userName']}@${project.properties['db.tnsName']}' schema is completed successfully..."
        println '***********************************************'
        println '*                                             *'
        println "*      '${description}' action is done...       *"
        println '*                                             *'
        println '***********************************************\n'
    }
}