From 616a5b5386e8ff0654b8727f376b58938865ead0 Mon Sep 17 00:00:00 2001 From: Mathieu Dalbin Date: Tue, 6 Feb 2024 08:58:06 +0100 Subject: [PATCH 1/8] Missing sortBuildListAsMap function for zCEE2 script (#465) * Implemented the missing sortBuildListAsMap function Signed-off-by: Mathieu Dalbin --- build-conf/zCEE2.properties | 3 ++- languages/zCEE2.groovy | 23 +++++++++++--------- utilities/BuildUtilities.groovy | 37 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/build-conf/zCEE2.properties b/build-conf/zCEE2.properties index f93d9c9f..5de48909 100644 --- a/build-conf/zCEE2.properties +++ b/build-conf/zCEE2.properties @@ -2,7 +2,7 @@ # # Comma separated list of required build properties for zCEE3.groovy -zcee2_requiredBuildProperties=zcee2_zconbtPath,zcee2_JAVA_HOME +zcee2_requiredBuildProperties=zcee2_zconbtPath # # Absolute path to zconbt executable on z/OS UNIX System Services @@ -12,6 +12,7 @@ zcee2_zconbtPath= # # Java installation used by the zconbt utility # for instance: /usr/lpp/java/J8.0_64 +# if not set, the value of the JAVA_HOME environment variable will be used zcee2_JAVA_HOME= # diff --git a/languages/zCEE2.groovy b/languages/zCEE2.groovy index c77a21e7..7eabb54d 100644 --- a/languages/zCEE2.groovy +++ b/languages/zCEE2.groovy @@ -15,7 +15,7 @@ import java.nio.file.*; buildUtils.assertBuildProperties(props.zcee2_requiredBuildProperties) // create updated build map, removing duplicates in case of PROJECT input Type -HashMap updatedBuildMap = new HashMap() +HashMap updatedBuildList = new HashMap() println("** Streamlining the build list to remove duplicates") argMap.buildList.each { buildFile -> @@ -36,27 +36,27 @@ argMap.buildList.each { buildFile -> } } if (projectDirFound) { - updatedBuildMap.put(projectDir.getPath(), "PROJECT") + updatedBuildList.putIfAbsent(projectDir.getPath(), "PROJECT") } else { if (props.verbose) println("!* No project directory found for file '${buildFile}'. Skipping...") } } else { - updatedBuildMap.put(buildFile, inputType); + updatedBuildList.put(buildFile, inputType); } } else { println("!* No Input Type mapping for file ${buildFile}, skipping it...") } } -println("** Building ${updatedBuildMap.size()} API ${updatedBuildMap.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script") +println("** Building ${updatedBuildList.size()} API ${updatedBuildList.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script") // sort the build list based on build file rank if provided -HashMap sortedMap = buildUtils.sortBuildMap(updatedBuildMap, 'zcee2_fileBuildRank') +HashMap sortedList = buildUtils.sortBuildListAsMap(updatedBuildList, 'zcee2_fileBuildRank') int currentBuildFileNumber = 1 // iterate through build list -sortedMap.each { buildFile, inputType -> - println "*** (${currentBuildFileNumber++}/${sortedMap.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile" +sortedList.each { buildFile, inputType -> + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile" String parameters = "" String outputDir = "" @@ -122,12 +122,15 @@ sortedMap.each { buildFile, inputType -> StringBuffer shellOutput = new StringBuffer() StringBuffer shellError = new StringBuffer() - String JAVA_HOME = props.getFileProperty('zcee2_JAVA_HOME', buildFile) + String javaHome = props.getFileProperty('zcee2_JAVA_HOME', buildFile) + if (!javaHome) { + javaHome = System.getenv("JAVA_HOME") + } ProcessBuilder cmd = new ProcessBuilder(zconbtPath, parameters); Map env = cmd.environment(); - env.put("JAVA_HOME", JAVA_HOME); - env.put("PATH", JAVA_HOME + "/bin" + ";" + env.get("PATH")) + env.put("JAVA_HOME", javaHome); + env.put("PATH", javaHome + "/bin" + ":" + env.get("PATH")) Process process = cmd.start() process.consumeProcessOutput(shellOutput, shellError) process.waitFor() diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 66f0b883..3ce63aef 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -285,6 +285,43 @@ def sortBuildList(List buildList, String rankPropertyName) { return sortedList } +/* + * sortBuildListAsMap - sorts a build List stored as Map by rank property values + */ +def sortBuildListAsMap(HashMap buildMap, String rankPropertyName) { + HashMap sortedMap = [:] + TreeMap> rankings = new TreeMap>() + HashMap unranked = new HashMap() + + // sort buildFiles by rank + buildMap.each { buildFile, inputType -> + String rank = props.getFileProperty(rankPropertyName, buildFile) + if (rank) { + Integer rankNum = rank.toInteger() + HashMap ranking = rankings.get(rankNum) + if (!ranking) { + ranking = new HashMap() + rankings.put(rankNum, ranking) + } + ranking.put(buildFile, inputType) + } else { + unranked.put(buildFile, inputType) + } + } + + // loop through rank keys adding sub lists (TreeMap automatically sorts keySet) + rankings.keySet().each { key -> + HashMap ranking = rankings.get(key) + if (ranking) + sortedMap.putAll(ranking) + } + + // finally add unranked buildFiles + sortedMap.putAll(unranked) + + return sortedMap +} + /* * updateBuildResult - used by language scripts to update the build result after a build step */ From f4e4c9b812037212f60ee4a070ee04b384470323 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 14 Feb 2024 09:30:53 +0100 Subject: [PATCH 2/8] Provide a summary of error messages in pipeline builds (#472) * Provide a summary of error messages in pipeline builds scenarios Signed-off-by: Dennis Behm --- build.groovy | 6 ++++++ utilities/BuildUtilities.groovy | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/build.groovy b/build.groovy index d2bb12ad..01d82649 100644 --- a/build.groovy +++ b/build.groovy @@ -793,7 +793,13 @@ def finalizeBuildProcess(Map args) { println("** Build State : $state") if (props.preview) println("** Build ran in preview mode.") println("** Total files processed : ${args.count}") + if (props.errorSummary) { + println("** Summary of error messages") + println("${props.errorSummary}") + } println("** Total build time : $duration\n") + + // if error occurred signal process error if (props.error) diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 3ce63aef..cf59f3c4 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -340,7 +340,8 @@ def updateBuildResult(Map args) { if (args.errorMsg) { buildResult.setStatus(buildResult.ERROR) buildResult.addProperty("error", args.errorMsg) - + errorSummary = (props.errorSummary) ? "${props.errorSummary} ${args.errorMsg}\n" : " ${args.errorMsg}\n" + props.put("errorSummary", "$errorSummary") } // add warning message, but keep result status From 3ad21d3ca12f4dc61ae03269404931c2433df81c Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 15 Feb 2024 14:35:06 +0100 Subject: [PATCH 3/8] Generate external impacted application reports only for impacted applications (#475) Signed-off-by: Dennis Behm --- utilities/ReportingUtilities.groovy | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/utilities/ReportingUtilities.groovy b/utilities/ReportingUtilities.groovy index 08b972a7..a85014f7 100644 --- a/utilities/ReportingUtilities.groovy +++ b/utilities/ReportingUtilities.groovy @@ -104,7 +104,7 @@ def reportExternalImpacts(Set changedFiles){ /** * Used to inspect dbb collections for potential impacts, sub-method to reportExternalImpacts * - * Returns a collection of the identified potential impacts + * Returns a list of DBB collections containing the identified potential impacted logical files * */ @@ -160,20 +160,23 @@ def writeExternalImpactReports(List logicalImpactedFilesCollections, logicalImpactedFilesCollections.each{ collectionImpacts -> def List logicalImpactedFiles = collectionImpacts.getLogicalFiles() - def collectionName = collectionImpacts.getName() - - String encodedFileName = URLEncoder.encode("externalImpacts_${collectionName}.log", "UTF-8") - String impactListFileLoc = "${props.buildOutDir}/${encodedFileName}" - if (props.verbose) println("*** Writing report of external impacts to file $impactListFileLoc") - File impactListFile = new File(impactListFileLoc) - String enc = props.logEncoding ?: 'IBM-1047' - impactListFile.withWriter(enc) { writer -> - - // write message for each file - logicalImpactedFiles.each{ logicalFile -> - if (props.verbose) println("$indentationMsg Potential external impact found ${logicalFile.getLname()} (${logicalFile.getFile()}) in collection ${collectionName} ") - def impactRecord = "${logicalFile.getLname()} \t ${logicalFile.getFile()} \t ${collectionName}" - writer.write("$impactRecord\n") + + // write reports for collections containing impact information + if (logicalImpactedFiles.size() > 0) { + def collectionName = collectionImpacts.getName() + String encodedFileName = URLEncoder.encode("externalImpacts_${collectionName}.log", "UTF-8") + String impactListFileLoc = "${props.buildOutDir}/${encodedFileName}" + if (props.verbose) println("*** Writing report of external impacts to file $impactListFileLoc") + File impactListFile = new File(impactListFileLoc) + String enc = props.logEncoding ?: 'IBM-1047' + impactListFile.withWriter(enc) { writer -> + + // write message for each file + logicalImpactedFiles.each{ logicalFile -> + if (props.verbose) println("$indentationMsg Potential external impact found ${logicalFile.getLname()} (${logicalFile.getFile()}) in collection ${collectionName} ") + def impactRecord = "${logicalFile.getLname()} \t ${logicalFile.getFile()} \t ${collectionName}" + writer.write("$impactRecord\n") + } } } } @@ -194,15 +197,13 @@ def calculateConcurrentChanges(Set buildSet) { // initialize patterns List gitRefMatcherPatterns = createMatcherPatterns(props.reportConcurrentChangesGitBranchReferencePatterns) - + // obtain all current remote branches // TODO: Handle / Exclude branches from other repositories Set remoteBranches = new HashSet() props.applicationSrcDirs.split(",").each { dir -> dir = buildUtils.getAbsolutePath(dir) - if (gitUtils.isGitDir(dir)) { - remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) - } + remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) } // Run analysis for each remoteBranch, which matches the configured criteria From 11eb18b4fbc5bbc983968f4b360026c5f06907d3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Mar 2024 10:06:28 +0100 Subject: [PATCH 4/8] Validate existence of workspace and outDir folder (#473) * Validate existence of workspace and outDir folder Signed-off-by: Dennis Behm --- build.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/build.groovy b/build.groovy index 01d82649..620d7e72 100644 --- a/build.groovy +++ b/build.groovy @@ -352,6 +352,19 @@ def populateBuildProperties(def opts) { // assert workspace buildUtils.assertBuildProperties('workspace,outDir') + // validate tht workspace and outDir folders exist + [props.workspace, props.outDir].each { workDirectory -> + if (!(new File (workDirectory).exists())) { + println "!! The specified folder $workDirectory does not exist. Build exits." + System.exit(1) + } + } + + if (!(new File(props.outDir).canWrite())) { + println "!! User does not have WRITE permission to work output directory ${props.outDir}. Build exits." + System.exit(1) + } + // load build.properties def buildConf = "${zAppBuildDir}/build-conf" if (opts.v) println "** Loading property file ${buildConf}/build.properties" From 27d902a0eef4a4f0faf91d1f38077faf9d23f1f3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Mar 2024 10:07:59 +0100 Subject: [PATCH 5/8] Implement Syntax rules for really long IDENTIFY statements (#480) * Implement Syntax rules for really long IDENTIFY statements Signed-off-by: Dennis Behm --- utilities/BuildUtilities.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index cf59f3c4..54bb8b39 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -898,17 +898,21 @@ def generateIdentifyStatement(String buildFile, String dsProperty) { String shortGitHash = getShortGitHash(buildFile) if (shortGitHash != null) { - String identifyString = props.application + "/" + shortGitHash // IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc') - identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" + identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" if (identifyString.length() > maxRecordLength) { String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt" println(errorMsg) props.error = "true" updateBuildResult(errorMsg:errorMsg) return null - } else { + } else { + if (identifyStmt.length() > 71) { // Split IDENTIFY after col 71 + // See syntax rules: https://www.ibm.com/docs/en/zos/3.1.0?topic=reference-identify-statement + identifyStmt = identifyStmt.substring(0,71) + "\n " + identifyStmt.substring(71,identifyStmt.length()) + } + // return generated IDENTIFY statement return identifyStmt } } else { From 452864ae827482c0c08e36624173583a85fa0ecc Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Mar 2024 10:54:50 +0100 Subject: [PATCH 6/8] Option to validate system datasets (#471) * Validate System Datasets Signed-off-by: Dennis Behm --- build-conf/build.properties | 8 ++- build.groovy | 8 ++- docs/BUILD.md | 75 ++++++++++++++++++++- utilities/DatasetValidationUtilities.groovy | 66 ++++++++++++++++++ utilities/README.md | 68 +++++++++++++++++-- 5 files changed, 215 insertions(+), 10 deletions(-) create mode 100644 utilities/DatasetValidationUtilities.groovy diff --git a/build-conf/build.properties b/build-conf/build.properties index acff3409..802e8c56 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -16,11 +16,17 @@ # These properties files expect to contain centrally managed defaults # such as system datasets, language script specific settings # -buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.properties,BMS.properties,\ +buildPropFiles=${systemDatasets},dependencyReport.properties,Assembler.properties,BMS.properties,\ MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\ LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\ CRB.properties,zCEE3.properties,zCEE2.properties +# +# Comma separated list of property files defining system datasets +# This property is validated during the initialization process of zAppBuild +# +systemDatasets=datasets.properties + # # Comma separated list of default application configuration property files to load # Supports both relative path (to zAppBuild/build-conf/) and absolute path diff --git a/build.groovy b/build.groovy index 620d7e72..24f418ad 100644 --- a/build.groovy +++ b/build.groovy @@ -20,6 +20,7 @@ import groovy.cli.commons.* @Field def reportingUtils= loadScript(new File("utilities/ReportingUtilities.groovy")) @Field def filePropUtils= loadScript(new File("utilities/FilePropUtilities.groovy")) @Field def dependencyScannerUtils= loadScript(new File("utilities/DependencyScannerUtilities.groovy")) +@Field def validationUtils= loadScript(new File("utilities/DatasetValidationUtilities.groovy")) @Field String hashPrefix = ':githash:' @Field String giturlPrefix = ':giturl:' @Field String gitchangedfilesPrefix = ':gitchangedfiles:' @@ -268,6 +269,7 @@ options: cli.r(longOpt:'reset', 'Deletes the dependency collections and build result group from the MetadataStore') cli.v(longOpt:'verbose', 'Flag to turn on script trace') cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands') + cli.cd(longOpt:'checkDatasets', 'Optional flag to validate the presense of the defined system datasets. ') // scan options cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)') @@ -275,7 +277,7 @@ options: cli.sl(longOpt:'scanLoad', 'Flag indicating to only scan load modules for application without building anything') cli.sa(longOpt:'scanAll', 'Flag indicating to scan both source files and load modules for application without building anything') - // web application credentials (overrides properties in build.properties) + // DBB metadatastore credentials (overrides properties in build.properties) cli.url(longOpt:'url', args:1, 'Db2 JDBC URL for the MetadataStore. Example: jdbc:db2:') cli.id(longOpt:'id', args:1, 'Db2 user id for the MetadataStore') cli.pw(longOpt:'pw', args:1, 'Db2 password (encrypted with DBB Password Utility) for the MetadataStore') @@ -469,6 +471,7 @@ def populateBuildProperties(def opts) { if (opts.b) props.baselineRef = opts.b if (opts.m) props.mergeBuild = 'true' if (opts.pv) props.preview = 'true' + if (opts.cd) props.checkDatasets = 'true' // scan options if (opts.s) props.scanOnly = 'true' @@ -541,6 +544,9 @@ def populateBuildProperties(def opts) { if(props.reportExternalImpactsAnalysisDepths) assert (props.reportExternalImpactsAnalysisDepths == 'simple' || props.reportExternalImpactsAnalysisDepths == 'deep' ) : "*! Build Property props.reportExternalImpactsAnalysisDepths has an invalid value" if(props.baselineRef) assert (props.impactBuild) : "*! Build Property props.baselineRef is exclusive to an impactBuild scenario" + // Validate system datasets + if (props.checkDatasets && props.systemDatasets) validationUtils.validateSystemDatasets(props.systemDatasets, props.verbose) + // Print all build properties + some envionment variables if (props.verbose) { println("java.version="+System.getProperty("java.runtime.version")) diff --git a/docs/BUILD.md b/docs/BUILD.md index 541d4317..fe196c01 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -222,7 +222,7 @@ build options: -cco,--cccOptions Headless Code Coverage Collector Options -re,--reportExternalImpacts Flag to activate analysis and report of external impacted files within DBB collections - + -cd,--checkDatasets Flag to enable validation of the defined system dataset definitions. Db2 MetadataStore configuration options -url,--url Db2 JDBC URL for the MetadataStore. @@ -256,6 +256,7 @@ utility options - [Perform a Scan Source build](#perform-a-scan-source-build) - [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build) - [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties) +- [Validate System Datasets](#validate-system-datasets) ### Build a Single Program @@ -2094,3 +2095,75 @@ required props = linkedit_srcPDS,linkedit_objPDS,linkedit_loadPDS,linkedit_linkE +### Validate System Datasets + +During the initialization phase of the build, a validation of the defined system datasets can be performed. The system datasets are configured with the build property `systemDatasets` in [build-conf/build.properties](../build-conf/build.properties), which contains one or multiple references to build property files defining key-value pairs listing the system datasets. In zAppBuild the default file is called [datasets.properties](../build-conf/datasets.properties) managed in the `build-conf` folder. + +To enable validation of system datasets specify the option `--checkDatasets`. It is available in any build scenario. Be aware that this functionality is also available as a stand-alone script and find the instructions [here](../utilities/README.md#dataset-validation-utilities) + +``` +groovyz dbb-zappbuild/build.groovy \ + --workspace /var/dbb/dbb-zappbuild/samples \ + --hlq DBB.ZAPP.CLEAN.MASTER \ + --workDir /var/dbb/out/MortgageApplication \ + --application MortgageApplication \ + --logEncoding UTF-8 \ + --impactBuild \ + --verbose \ + --checkDatasets +``` +
+ Build log + +``` +** Build start at 20210622.082942.029 +** Input args = /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --impactBuild --verbose --propOverwrite mainBuildBranch=develop +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Assembler.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/BMS.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/MFS.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PSBgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/DBDgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ACBgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Cobol.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/LinkEdit.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PLI.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ZunitConfig.properties +** appConf = /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/file.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/BMS.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/Cobol.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/LinkEdit.properties +** Overwriting build property mainBuildBranch from cli argument --propOverwrite with value develop +** The dataset PLI.V5R2.SIBMZCMP referenced for property IBMZPLI_V52 was found. +*! No dataset defined for property IBMZPLI_V51 specified in /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties. +** The dataset WMQ.V9R2M4.SCSQPLIC referenced for property SCSQPLIC was found. +** The dataset COBOL.V6R1.SIGYCOMP referenced for property SIGYCOMP_V6 was found. +** The dataset CICSTS.V5R4.CICS.SDFHCOB referenced for property SDFHCOB was found. +*! No dataset defined for property SIGYCOMP_V4 specified in /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties. +** The dataset HLASM.SASMMOD1 referenced for property SASMMOD1 was found. +** The dataset SYS1.MACLIB referenced for property MACLIB was found. +** The dataset PDTCC.V1R8.SIPVMODA referenced for property PDTCCMOD was found. +** The dataset CICSTS.V5R4.CICS.SDFHLOAD referenced for property SDFHLOAD was found. +** The dataset CICSTS.V5R4.CICS.SDFHMAC referenced for property SDFHMAC was found. +** The dataset CEE.SCEEMAC referenced for property SCEEMAC was found. +** The dataset WMQ.V9R2M4.SCSQCOBC referenced for property SCSQCOBC was found. +** The dataset IMS.V15R1.SDFSMAC referenced for property SDFSMAC was found. +** The dataset RDZ.V14R1.SFELLOAD referenced for property SFELLOAD was found. +** The dataset DBC0CFG.DB2.V12.SDSNLOAD referenced for property SDSNLOAD was found. +** The dataset CICSTS.V5R4.CICS.SDFHPL1 referenced for property SDFHPL1 was found. +** The dataset WMQ.V9R2M4.SCSQLOAD referenced for property SCSQLOAD was found. +** The dataset IMSCFG.IMSC.REFERAL referenced for property REFERAL was found. +** The dataset DEBUG.V14R1.SEQAMOD referenced for property SEQAMOD was found. +** The dataset DBC0CFG.SDSNEXIT referenced for property SDSNEXIT was found. +** The dataset IMS.V15R1.SDFSRESL referenced for property SDFSRESL was found. +** The dataset RATCFG.ZUNIT.SBZUSAMP referenced for property SBZUSAMP was found. +** The dataset CEE.SCEELKED referenced for property SCEELKED was found. +..... // lists of all build properties +... +... +``` + +
+ + diff --git a/utilities/DatasetValidationUtilities.groovy b/utilities/DatasetValidationUtilities.groovy new file mode 100644 index 00000000..bc91340b --- /dev/null +++ b/utilities/DatasetValidationUtilities.groovy @@ -0,0 +1,66 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.build.* +import groovy.transform.* +import groovy.cli.commons.* +import java.util.Properties +import com.ibm.jzos.ZFile + +// define script properties + +@Field BuildProperties props = BuildProperties.getInstance() + +validateDatasets(args) + +def validateSystemDatasets(String propertyFiles, String verbose) { + + propertyFiles.split(",").each { propertyFile -> + + // load property file using java.util.Properties + def properties = new Properties() + + // convert to absolute path based on zAppBuild conventions and structure + if (props && !propertyFile.startsWith('/')) propertyFile = "${props.zAppBuildDir}/build-conf/$propertyFile" + + File propFile = new File(propertyFile) + if (propFile.exists()) { + + propFile.withInputStream { properties.load(it) } + properties.each { key,dataset -> + if (dataset) { + if (ZFile.dsExists("'$dataset'")) { + if (verbose.toBoolean()) println "** The dataset $dataset referenced for property $key was found." + } else { + println "*! The dataset $dataset referenced for property $key was not found. Process exits." + System.exit(1) + } + } else { + if (verbose.toBoolean()) println "*! No dataset defined for property $key specified in $propertyFile." + } + } + } else { + println "*!* The specified $propertyFile (in the list [$propertyFiles]) does not exist." + } + } +} + +def validateDatasets(String[] cliArgs) +{ + def cli = new CliBuilder(usage: "DatasetValidationUtilites.groovy [options]", header: '', stopAtNonOption: false) + cli.d(longOpt:'systemDatasetDefinition', args:1, required:true, 'List of property files containing system dataset definitions.') + cli.h(longOpt:'help', 'Flag to print the Help message.') + + def opts = cli.parse(cliArgs) + + // if opt parse fail exit. + if (! opts) { + System.exit(1) + } + + if (opts.help) + { + cli.usage() + System.exit(0) + } + + validateSystemDatasets(opts.d, "true") +} \ No newline at end of file diff --git a/utilities/README.md b/utilities/README.md index 254a1ea4..a6c04522 100644 --- a/utilities/README.md +++ b/utilities/README.md @@ -3,10 +3,64 @@ This folder contains common utility and helper files used by the zAppBuild `buil File | Description --- | --- -BindUtilities.groovy | Use for the DB2 binding. -BuildReportUtilities.groovy | Helper to populate additional build report entries. -BuildUtilities.groovy | Common build utility methods. -DependencyScannerUtilities.groovy | Populates the DependencyScannerRegistry and returns the mapped dependency scanner for a build file. -FilePropUtilities.groovy | Helper util to load and validate file level properties overrides. -GitUtilities.groovy | Git command methods. -ImpactUtilities.groovy | Methods used for ImpactBuilds. +[BindUtilities.groovy](BindUtilities.groovy) | Use for the DB2 binding. +[BuildReportUtilities.groovy](BuildReportUtilities.groovy) | Helper to populate additional build report entries. +[BuildUtilities.groovy](BuildUtilities.groovy) | Common build utility methods. +[DatasetValidationUtilities.groovy](DatasetValidationUtilities.groovy) | Helper to validate that system dataset definitions are correctly defined. +[DependencyScannerUtilities.groovy](DependencyScannerUtilities.groovy) | Populates the DependencyScannerRegistry and returns the mapped dependency scanner for a build file. +[FilePropUtilities.groovy](FilePropUtilities.groovy) | Helper util to load and validate file level properties overrides. +[GitUtilities.groovy](GitUtilities.groovy) | Git command methods. +[ImpactUtilities.groovy](ImpactUtilities.groovy) | Methods used for ImpactBuilds. + +## Stand-alone utils + +A few utilities are used both in the zAppBuild build process, but can also be invoked stand-alone: +* Dataset Validation Utilities +* Bind Utilities + +### Dataset Validation Utilities + +The [DatasetValidationUtilities.groovy](DatasetValidationUtilities.groovy) can be used as a standalone script to validate that the configured system datasets such as the [dataset.properties](../build-conf/datasets.properties) are available on the system where this script is executed. Using the script as a standalone utility targets situations where builds runs into allocation exceptions such as `BGZTK0016E An error occurred running BPXWDYN command 'alloc dd(TASKLIB) dsn(COBOL.V6R1.SIGYCOM) shr'` and helps the build script administrator to customize zAppBuild. + +#### Usage + +``` +groovyz /u/ibmuser/zAppBuild/utilities/DatasetValidationUtilities.groovy --help + +usage: DatasetValidationUtilites.groovy [options] + + -d,--systemDatasetDefinition List of property files containing + system dataset definitions. + -h,--help Flag to print the Help message. +``` + +#### Sample invocation + +``` +groovyz /u/ibmuser/zAppBuild/utilities/DatasetValidationUtilities.groovy -d /var/dbb/dbb-zappbuild-config/datasets.properties + +** The dataset PLI.V5R2.SIBMZCMP referenced for property IBMZPLI_V52 was found. +*! No dataset defined for property IBMZPLI_V51 specified in /var/dbb/dbb-zappbuild-config/datasets.properties. +** The dataset WMQ.V9R2M4.SCSQPLIC referenced for property SCSQPLIC was found. +** The dataset COBOL.V6R1.SIGYCOMP referenced for property SIGYCOMP_V6 was found. +** The dataset CICSTS.V5R4.CICS.SDFHCOB referenced for property SDFHCOB was found. +*! No dataset defined for property SIGYCOMP_V4 specified in /var/dbb/dbb-zappbuild-config/datasets.properties. +** The dataset HLASM.SASMMOD1 referenced for property SASMMOD1 was found. +** The dataset SYS1.MACLIB referenced for property MACLIB was found. +** The dataset PDTCC.V1R8.SIPVMODA referenced for property PDTCCMOD was found. +** The dataset CICSTS.V5R4.CICS.SDFHLOAD referenced for property SDFHLOAD was found. +** The dataset CICSTS.V5R4.CICS.SDFHMAC referenced for property SDFHMAC was found. +** The dataset CEE.SCEEMAC referenced for property SCEEMAC was found. +** The dataset WMQ.V9R2M4.SCSQCOBC referenced for property SCSQCOBC was found. +** The dataset IMS.V15R1.SDFSMAC referenced for property SDFSMAC was found. +** The dataset RDZ.V14R1.SFELLOAD referenced for property SFELLOAD was found. +** The dataset DBC0CFG.DB2.V12.SDSNLOAD referenced for property SDSNLOAD was found. +** The dataset CICSTS.V5R4.CICS.SDFHPL1 referenced for property SDFHPL1 was found. +** The dataset WMQ.V9R2M4.SCSQLOAD referenced for property SCSQLOAD was found. +** The dataset IMSCFG.IMSC.REFERAL referenced for property REFERAL was found. +** The dataset DEBUG.V14R1.SEQAMOD referenced for property SEQAMOD was found. +** The dataset DBC0CFG.SDSNEXIT referenced for property SDSNEXIT was found. +** The dataset IMS.V15R1.SDFSRESL referenced for property SDFSRESL was found. +** The dataset RATCFG.ZUNIT.SBZUSAMP referenced for property SBZUSAMP was found. +** The dataset CEE.SCEELKED referenced for property SCEELKED was found. +``` From 0d533dc68ac0f50d1d75f9753ca7fde46c486198 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Mar 2024 14:33:12 +0100 Subject: [PATCH 7/8] Fix syntax error in resetBuild testscript (#482) Signed-off-by: Dennis Behm --- test/testScripts/resetBuild.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/test/testScripts/resetBuild.groovy b/test/testScripts/resetBuild.groovy index 419ef7d8..59d1a6d0 100644 --- a/test/testScripts/resetBuild.groovy +++ b/test/testScripts/resetBuild.groovy @@ -4,6 +4,7 @@ import com.ibm.dbb.* import com.ibm.dbb.build.* @Field BuildProperties props = BuildProperties.getInstance() +@Field def assertionList = [] println "\n**************************************************************" println "** Executing test script ${this.class.getName()}.groovy" From fb75b543e1b29d57c298a417e7978038c3618081 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Mar 2024 14:34:07 +0100 Subject: [PATCH 8/8] Workspace and outDir directory validation (#481) Signed-off-by: Dennis Behm --- build.groovy | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/build.groovy b/build.groovy index 24f418ad..59729618 100644 --- a/build.groovy +++ b/build.groovy @@ -354,15 +354,14 @@ def populateBuildProperties(def opts) { // assert workspace buildUtils.assertBuildProperties('workspace,outDir') - // validate tht workspace and outDir folders exist - [props.workspace, props.outDir].each { workDirectory -> - if (!(new File (workDirectory).exists())) { - println "!! The specified folder $workDirectory does not exist. Build exits." - System.exit(1) - } + // Validate that workspace exists + if (!(new File (props.workspace).exists())) { + println "!! The specified workspace folder ${props.workspace} does not exist. Build exits." + System.exit(1) } - if (!(new File(props.outDir).canWrite())) { + // Check read/write permission of specified out/log dir if already existing + if (new File (props.outDir).exists() && !(new File(props.outDir).canWrite())) { println "!! User does not have WRITE permission to work output directory ${props.outDir}. Build exits." System.exit(1) }