Merge pull request #483 from IBM/develop

Release zAppBuild 3.7.0

build-conf/build.properties

@@ -16,11 +16,17 @@
 # These properties files expect to contain centrally managed defaults
 #  such as system datasets, language script specific settings
 #
-buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.properties,BMS.properties,\
+buildPropFiles=${systemDatasets},dependencyReport.properties,Assembler.properties,BMS.properties,\
 MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\
 LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\
 CRB.properties,zCEE3.properties,zCEE2.properties
 
+#
+# Comma separated list of property files defining system datasets
+# This property is validated during the initialization process of zAppBuild  
+#
+systemDatasets=datasets.properties
+
 #
 # Comma separated list of default application configuration property files to load
 # Supports both relative path (to zAppBuild/build-conf/) and absolute path

build-conf/zCEE2.properties

@@ -2,7 +2,7 @@
 
 #
 # Comma separated list of required build properties for zCEE3.groovy
-zcee2_requiredBuildProperties=zcee2_zconbtPath,zcee2_JAVA_HOME
+zcee2_requiredBuildProperties=zcee2_zconbtPath
 
 #
 # Absolute path to zconbt executable on z/OS UNIX System Services
@@ -12,6 +12,7 @@ zcee2_zconbtPath=
 #
 # Java installation used by the zconbt utility
 # for instance: /usr/lpp/java/J8.0_64
+# if not set, the value of the JAVA_HOME environment variable will be used
 zcee2_JAVA_HOME=
 
 #

build.groovy

@@ -20,6 +20,7 @@ import groovy.cli.commons.*
 @Field def reportingUtils= loadScript(new File("utilities/ReportingUtilities.groovy"))
 @Field def filePropUtils= loadScript(new File("utilities/FilePropUtilities.groovy"))
 @Field def dependencyScannerUtils= loadScript(new File("utilities/DependencyScannerUtilities.groovy"))
+@Field def validationUtils= loadScript(new File("utilities/DatasetValidationUtilities.groovy"))
 @Field String hashPrefix = ':githash:'
 @Field String giturlPrefix = ':giturl:'
 @Field String gitchangedfilesPrefix = ':gitchangedfiles:'
@@ -268,14 +269,15 @@ options:
 	cli.r(longOpt:'reset', 'Deletes the dependency collections and build result group from the MetadataStore')
 	cli.v(longOpt:'verbose', 'Flag to turn on script trace')
 	cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands')
+	cli.cd(longOpt:'checkDatasets', 'Optional flag to validate the presense of the defined system datasets. ')
 
 	// scan options
 	cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)')
 	cli.ss(longOpt:'scanSource', 'Flag indicating to only scan source files for application without building anything')
 	cli.sl(longOpt:'scanLoad', 'Flag indicating to only scan load modules for application without building anything')
 	cli.sa(longOpt:'scanAll', 'Flag indicating to scan both source files and load modules for application without building anything')
 
-	// web application credentials (overrides properties in build.properties)
+	// DBB metadatastore credentials (overrides properties in build.properties)
 	cli.url(longOpt:'url', args:1, 'Db2 JDBC URL for the MetadataStore. Example: jdbc:db2:<Db2 server location>')
 	cli.id(longOpt:'id', args:1, 'Db2 user id for the MetadataStore')
 	cli.pw(longOpt:'pw', args:1,  'Db2 password (encrypted with DBB Password Utility) for the MetadataStore')
@@ -352,6 +354,18 @@ def populateBuildProperties(def opts) {
 	// assert workspace
 	buildUtils.assertBuildProperties('workspace,outDir')
 
+	// Validate that workspace exists 
+	if (!(new File (props.workspace).exists())) {
+		println "!! The specified workspace folder ${props.workspace} does not exist. Build exits."
+		System.exit(1)
+	}
+
+	// Check read/write permission of specified out/log dir if already existing 
+	if (new File (props.outDir).exists() && !(new File(props.outDir).canWrite())) {
+		println "!! User does not have WRITE permission to work output directory ${props.outDir}. Build exits."
+		System.exit(1)
+	}
+
 	// load build.properties
 	def buildConf = "${zAppBuildDir}/build-conf"
 	if (opts.v) println "** Loading property file ${buildConf}/build.properties"
@@ -456,6 +470,7 @@ def populateBuildProperties(def opts) {
 	if (opts.b) props.baselineRef = opts.b
 	if (opts.m) props.mergeBuild = 'true'
 	if (opts.pv) props.preview = 'true'
+	if (opts.cd) props.checkDatasets = 'true'
 
 	// scan options
 	if (opts.s) props.scanOnly = 'true'
@@ -528,6 +543,9 @@ def populateBuildProperties(def opts) {
 	if(props.reportExternalImpactsAnalysisDepths) assert (props.reportExternalImpactsAnalysisDepths == 'simple' || props.reportExternalImpactsAnalysisDepths == 'deep' ) : "*! Build Property props.reportExternalImpactsAnalysisDepths has an invalid value"
 	if(props.baselineRef) assert (props.impactBuild) : "*! Build Property props.baselineRef is exclusive to an impactBuild scenario"
 
+	// Validate system datasets
+	if (props.checkDatasets && props.systemDatasets) validationUtils.validateSystemDatasets(props.systemDatasets, props.verbose)
+
 	// Print all build properties + some envionment variables 
 	if (props.verbose) {
 		println("java.version="+System.getProperty("java.runtime.version"))
@@ -793,7 +811,13 @@ def finalizeBuildProcess(Map args) {
 	println("** Build State : $state")
 	if (props.preview) println("** Build ran in preview mode.")
 	println("** Total files processed : ${args.count}")
+	if (props.errorSummary) {
+		println("** Summary of error messages")
+		println("${props.errorSummary}")
+	}
 	println("** Total build time  : $duration\n")
+
+
 
 	// if error occurred signal process error
 	if (props.error)

docs/BUILD.md

@@ -222,7 +222,7 @@ build options:
  -cco,--cccOptions            Headless Code Coverage Collector Options
 
  -re,--reportExternalImpacts  Flag to activate analysis and report of external impacted files within DBB collections
-
+ -cd,--checkDatasets          Flag to enable validation of the defined system dataset definitions.
 
 Db2 MetadataStore configuration options
  -url,--url <arg>             Db2 JDBC URL for the MetadataStore.
@@ -256,6 +256,7 @@ utility options
 - [Perform a Scan Source build](#perform-a-scan-source-build)
 - [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build)
 - [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties)
+- [Validate System Datasets](#validate-system-datasets)
 
 <!-- /TOC -->
 ### Build a Single Program
@@ -2094,3 +2095,75 @@ required props = linkedit_srcPDS,linkedit_objPDS,linkedit_loadPDS,linkedit_linkE
 </details>
 
 
+### Validate System Datasets
+
+During the initialization phase of the build, a validation of the defined system datasets can be performed. The system datasets are configured with the build property `systemDatasets` in [build-conf/build.properties](../build-conf/build.properties), which contains one or multiple references to build property files defining key-value pairs listing the system datasets. In zAppBuild the default file is called [datasets.properties](../build-conf/datasets.properties) managed in the `build-conf` folder.
+
+To enable validation of system datasets specify the option `--checkDatasets`. It is available in any build scenario. Be aware that this functionality is also available as a stand-alone script and find the instructions [here](../utilities/README.md#dataset-validation-utilities)
+
+```
+groovyz dbb-zappbuild/build.groovy \
+                      --workspace /var/dbb/dbb-zappbuild/samples \
+                      --hlq DBB.ZAPP.CLEAN.MASTER \
+                      --workDir /var/dbb/out/MortgageApplication \
+                      --application MortgageApplication \
+                      --logEncoding UTF-8 \
+                      --impactBuild \
+                      --verbose \
+                      --checkDatasets
+```
+<details>
+  <summary>Build log</summary>
+
+```
+** Build start at 20210622.082942.029
+** Input args = /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --impactBuild --verbose --propOverwrite mainBuildBranch=develop
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Assembler.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/BMS.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/MFS.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PSBgen.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/DBDgen.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ACBgen.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Cobol.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/LinkEdit.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PLI.properties
+** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ZunitConfig.properties
+** appConf = /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf
+** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/file.properties
+** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/BMS.properties
+** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/Cobol.properties
+** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/LinkEdit.properties
+** Overwriting build property mainBuildBranch from cli argument --propOverwrite with value develop
+** The dataset PLI.V5R2.SIBMZCMP referenced for property IBMZPLI_V52 was found.
+*! No dataset defined for property IBMZPLI_V51 specified in /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties.
+** The dataset WMQ.V9R2M4.SCSQPLIC referenced for property SCSQPLIC was found.
+** The dataset COBOL.V6R1.SIGYCOMP referenced for property SIGYCOMP_V6 was found.
+** The dataset CICSTS.V5R4.CICS.SDFHCOB referenced for property SDFHCOB was found.
+*! No dataset defined for property SIGYCOMP_V4 specified in /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties.
+** The dataset HLASM.SASMMOD1 referenced for property SASMMOD1 was found.
+** The dataset SYS1.MACLIB referenced for property MACLIB was found.
+** The dataset PDTCC.V1R8.SIPVMODA referenced for property PDTCCMOD was found.
+** The dataset CICSTS.V5R4.CICS.SDFHLOAD referenced for property SDFHLOAD was found.
+** The dataset CICSTS.V5R4.CICS.SDFHMAC referenced for property SDFHMAC was found.
+** The dataset CEE.SCEEMAC referenced for property SCEEMAC was found.
+** The dataset WMQ.V9R2M4.SCSQCOBC referenced for property SCSQCOBC was found.
+** The dataset IMS.V15R1.SDFSMAC referenced for property SDFSMAC was found.
+** The dataset RDZ.V14R1.SFELLOAD referenced for property SFELLOAD was found.
+** The dataset DBC0CFG.DB2.V12.SDSNLOAD referenced for property SDSNLOAD was found.
+** The dataset CICSTS.V5R4.CICS.SDFHPL1 referenced for property SDFHPL1 was found.
+** The dataset WMQ.V9R2M4.SCSQLOAD referenced for property SCSQLOAD was found.
+** The dataset IMSCFG.IMSC.REFERAL referenced for property REFERAL was found.
+** The dataset DEBUG.V14R1.SEQAMOD referenced for property SEQAMOD was found.
+** The dataset DBC0CFG.SDSNEXIT referenced for property SDSNEXIT was found.
+** The dataset IMS.V15R1.SDFSRESL referenced for property SDFSRESL was found.
+** The dataset RATCFG.ZUNIT.SBZUSAMP referenced for property SBZUSAMP was found.
+** The dataset CEE.SCEELKED referenced for property SCEELKED was found.
+..... // lists of all build properties
+...
+...
+```
+
+</details>
+
+

languages/zCEE2.groovy

@@ -15,7 +15,7 @@ import java.nio.file.*;
 buildUtils.assertBuildProperties(props.zcee2_requiredBuildProperties)
 
 // create updated build map, removing duplicates in case of PROJECT input Type
-HashMap<String, String> updatedBuildMap = new HashMap<String, String>()
+HashMap<String, String> updatedBuildList = new HashMap<String, String>()
 
 println("** Streamlining the build list to remove duplicates")
 argMap.buildList.each { buildFile ->
@@ -36,27 +36,27 @@ argMap.buildList.each { buildFile ->
                 }
             }
             if (projectDirFound) {
-                updatedBuildMap.put(projectDir.getPath(), "PROJECT")
+                updatedBuildList.putIfAbsent(projectDir.getPath(), "PROJECT")
             } else {
                 if (props.verbose) println("!* No project directory found for file '${buildFile}'. Skipping...")
             }
         } else {
-            updatedBuildMap.put(buildFile, inputType);
+            updatedBuildList.put(buildFile, inputType);
         }
     } else {
         println("!* No Input Type mapping for file ${buildFile}, skipping it...")
     }
 }
 
-println("** Building ${updatedBuildMap.size()} API ${updatedBuildMap.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script")
+println("** Building ${updatedBuildList.size()} API ${updatedBuildList.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script")
 // sort the build list based on build file rank if provided
-HashMap<String, String> sortedMap = buildUtils.sortBuildMap(updatedBuildMap, 'zcee2_fileBuildRank')
+HashMap<String, String> sortedList = buildUtils.sortBuildListAsMap(updatedBuildList, 'zcee2_fileBuildRank')
 
 int currentBuildFileNumber = 1
 
 // iterate through build list
-sortedMap.each { buildFile, inputType ->
-    println "*** (${currentBuildFileNumber++}/${sortedMap.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile"
+sortedList.each { buildFile, inputType ->
+    println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile"
 
     String parameters = ""
     String outputDir = ""
@@ -122,12 +122,15 @@ sortedMap.each { buildFile, inputType ->
         StringBuffer shellOutput = new StringBuffer()
         StringBuffer shellError = new StringBuffer()
 
-        String JAVA_HOME = props.getFileProperty('zcee2_JAVA_HOME', buildFile)
+        String javaHome = props.getFileProperty('zcee2_JAVA_HOME', buildFile)
+        if (!javaHome) {
+            javaHome = System.getenv("JAVA_HOME")
+        }
 
         ProcessBuilder cmd = new ProcessBuilder(zconbtPath, parameters);
         Map<String, String> env = cmd.environment();
-        env.put("JAVA_HOME", JAVA_HOME);
-        env.put("PATH", JAVA_HOME + "/bin" + ";" + env.get("PATH"))
+        env.put("JAVA_HOME", javaHome);
+        env.put("PATH", javaHome + "/bin" + ":" + env.get("PATH"))
         Process process = cmd.start()
         process.consumeProcessOutput(shellOutput, shellError)
         process.waitFor()

test/testScripts/resetBuild.groovy

@@ -4,6 +4,7 @@ import com.ibm.dbb.*
 import com.ibm.dbb.build.*
 
 @Field BuildProperties props = BuildProperties.getInstance()
+@Field def assertionList = []
 
 println "\n**************************************************************"
 println "** Executing test script ${this.class.getName()}.groovy"

utilities/BuildUtilities.groovy

@@ -285,6 +285,43 @@ def sortBuildList(List<String> buildList, String rankPropertyName) {
 	return sortedList
 }
 
+/*
+ * sortBuildListAsMap - sorts a build List stored as Map by rank property values
+ */
+def sortBuildListAsMap(HashMap<String, String> buildMap, String rankPropertyName) {
+	HashMap<String, String> sortedMap = [:]
+	TreeMap<Integer,HashMap<String, String>> rankings = new TreeMap<Integer,HashMap<String, String>>()
+	HashMap<String, String> unranked = new HashMap<String, String>()
+
+	// sort buildFiles by rank
+	buildMap.each { buildFile, inputType ->
+		String rank = props.getFileProperty(rankPropertyName, buildFile)
+		if (rank) {
+			Integer rankNum = rank.toInteger()
+			HashMap<String, String> ranking = rankings.get(rankNum)
+			if (!ranking) {
+				ranking = new HashMap<String, String>()
+				rankings.put(rankNum, ranking)
+			}
+			ranking.put(buildFile, inputType)
+		} else {
+			unranked.put(buildFile, inputType)
+		}
+	}
+
+	// loop through rank keys adding sub lists (TreeMap automatically sorts keySet)
+	rankings.keySet().each { key ->
+		HashMap<String, String> ranking = rankings.get(key)
+		if (ranking)
+			sortedMap.putAll(ranking)
+	}
+
+	// finally add unranked buildFiles
+	sortedMap.putAll(unranked)
+
+	return sortedMap
+}
+
 /*
  * updateBuildResult - used by language scripts to update the build result after a build step
  */
@@ -303,7 +340,8 @@ def updateBuildResult(Map args) {
 		if (args.errorMsg) {
 			buildResult.setStatus(buildResult.ERROR)
 			buildResult.addProperty("error", args.errorMsg)
-
+			errorSummary = (props.errorSummary) ?  "${props.errorSummary}   ${args.errorMsg}\n" : "   ${args.errorMsg}\n"
+			props.put("errorSummary", "$errorSummary")
 		}
 
 		// add warning message, but keep result status
@@ -860,17 +898,21 @@ def generateIdentifyStatement(String buildFile, String dsProperty) {
 		String shortGitHash = getShortGitHash(buildFile)
 
 		if (shortGitHash != null) {
-
 			String identifyString = props.application + "/" + shortGitHash
 			//   IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc')
-			identifyStmt = "  " + "IDENTIFY ${member}(\'$identifyString\')"
+			identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')"
 			if (identifyString.length() > maxRecordLength) {
 				String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt"
 				println(errorMsg)
 				props.error = "true"
 				updateBuildResult(errorMsg:errorMsg)
 				return null
-			} else {
+			} else { 
+				if (identifyStmt.length() > 71) { // Split IDENTIFY after col 71
+					// See syntax rules: https://www.ibm.com/docs/en/zos/3.1.0?topic=reference-identify-statement
+					identifyStmt = identifyStmt.substring(0,71) + "\n " + identifyStmt.substring(71,identifyStmt.length())
+				}
+				// return generated IDENTIFY statement
 				return identifyStmt
 			}
 		} else {