Merge pull request #5 from MetLife/v1.1

V1.1

CHANGELOG.md

@@ -2,3 +2,9 @@
 
 # v1.0.0
 - Initial Release
+
+# v1.1.0
+- Removed scanner installation functionality. We now run the SRCCLR CI script from https://download.sourceclear.com/ci.sh. This will ensure we are always running the latest version of the SRCCLR scanner when using a local Azure DevOps Agent and will preclude us from needing root permissions to install the scanner.
+- Added the --recursive directive so that url and directory scans provide better coverage when applications use multiple package managers.
+- Set the CACHE_DIR to Agent.TempDirectory. CACHE_DIR is a feature for SCA to direct where the SCA scanner is downloaded to. The Agent.TempDirectory is cleaned out after every job, which is useful for users leveraging a local Azure DevOps agent and disk space fills up (default was /tmp).
+- Improved error handling for python tasks.

README.md

@@ -82,7 +82,3 @@ The location of the latest self-hosted agents is [here](https://github.com/micro
 ## References
 
 [Here](https://www.paraesthesia.com/archive/2020/02/25/tips-for-custom-azure-devops-build-tasks/) are some useful tips for developing tasks for Azure DevOps.
-
-## Feedback
-
-Send me mail at gattjoseph@hotmail.com

buildAndReleaseTask/package.json

@@ -15,7 +15,7 @@
     "azure-pipelines-tool-lib": "^0.13.2"
   },
   "devDependencies": {
-    "@types/node": "^14.14.11",
+    "@types/node": "^14.14.14",
     "@types/q": "^1.5.4"
   }
 }

buildAndReleaseTask/scascan.ts

@@ -8,6 +8,20 @@ import * as tl from 'azure-pipelines-task-lib/task';
 import * as trm from 'azure-pipelines-task-lib/toolrunner';
 import * as path from 'path';
 
+/**
+ * Get Agent.TempDirectory which is a temp folder that is cleaned after each pipeline job.
+ * This is where we will store the Veracode SCA tool so we do not take up too much disk space
+ * on self hosted agents.
+ */
+const tempPath: string = <string>tl.getVariable('Agent.TempDirectory');
+if (tempPath !== undefined) {
+    // Per https://help.veracode.com/r/c_sc_ci_script if we set
+    // the CACHE_DIR variable, we can direct where the files are downloaded to
+    tl.setVariable('CACHE_DIR', tempPath);
+    console.log(`CACHE_DIR: ${tempPath}`);
+}
+
+
 async function run(): Promise<void> {
     try {
         tl.setResourcePath(path.join(__dirname, 'task.json'));
@@ -48,93 +62,54 @@ async function run(): Promise<void> {
 
         //This only works on linux or MacOs Agents
         if (agentPlatform === 'linux' || agentPlatform === 'darwin') {
-
-            try {
-                // Is srcclr already installed?
-                const srcclrPath: string = tl.which('srcclr', true);
-                console.log('Found SCA Agent install here: ' + `${srcclrPath}`);
-
-            } catch {
-                // Install srcclr
-                const curlPath: string = tl.which('curl', true);
-                const shPath: string = tl.which('sh', true);
-
-                // Install SCA Agent
-                const curl: trm.ToolRunner = tl.tool(curlPath);
-                curl.arg('-sSL');
-                curl.arg('https://www.sourceclear.com/install');
-                const sh: trm.ToolRunner = tl.tool(shPath);
-                // On self-hosted agents this may not work if the agent is not running as root
-                const pipe: trm.ToolRunner = curl.pipeExecOutputToTool(sh);
-                const scaAgentInstall: number = await pipe.exec();
-                tl.setResult(tl.TaskResult.Succeeded, tl.loc('curlReturnCode', scaAgentInstall));
-
-            }
-
+
             // Test the environment to see which collectors are available, equivalent to 'srcclr test'
             if (testAgent === true) {
-                const scaAgentTest: trm.ToolRunner = tl.tool('srcclr');
-                scaAgentTest.arg('test');
-                const scaAgentTestResult: number = await scaAgentTest.exec();
-                tl.setResult(tl.TaskResult.Succeeded, tl.loc('bashReturnCode', scaAgentTestResult));
+                await testSCA();
             }
+
+            // Run the scan
+            await runScan(scanType, scanTarget);
+
+            // Find the python3 installation
+            const pythonPath: string = tl.which('python3');
 
-            // Scan against an artifact directory
-            if (scanType === 'directory') {
-                const scanDirectory: trm.ToolRunner = tl.tool('srcclr');
-                scanDirectory.arg('scan');
-                scanDirectory.arg(`${scanTarget}`);
-                scanDirectory.arg('--json');
-                scanDirectory.arg('scaresults.json');
-                const directoryResults: number = await scanDirectory.exec();
-                tl.setResult(tl.TaskResult.Succeeded, tl.loc('bashReturnCode', directoryResults));
-            // Scan against a URL - Need to make sure it begins with http(s)
-            } else if (scanType === 'url') {
-                const scanUrl: trm.ToolRunner = tl.tool('srcclr');
-                scanUrl.arg('scan');
-                scanUrl.arg('--url');
-                scanUrl.arg(`${scanTarget}`);
-                scanUrl.arg('--json');
-                scanUrl.arg('scaresults.json');
-                const urlResults: number = await scanUrl.exec();
-                tl.setResult(tl.TaskResult.Succeeded, tl.loc('bashReturnCode', urlResults));
-            // Scan a Docker image
-            } else if (scanType === 'image') {
-                const scanDockerImage: trm.ToolRunner = tl.tool('srcclr');
-                scanDockerImage.arg('scan');
-                scanDockerImage.arg('--image');
-                scanDockerImage.arg(`${scanTarget}`);
-                scanDockerImage.arg('--json');
-                scanDockerImage.arg('scaresults.json');
-                const dockerResults: number = await scanDockerImage.exec();
-                tl. setResult(tl.TaskResult.Succeeded, tl.loc('bashReturnCode', dockerResults));
+            try {
+                // Install junitparser
+                const python3: trm.ToolRunner = tl.tool(pythonPath);
+                python3.arg('-m');
+                python3.arg('pip');
+                python3.arg('install');
+                python3.arg('--upgrade');
+                python3.arg('pip');
+                python3.arg('junitparser');
+                // Run the command
+                await python3.exec();
+                tl.setResult(tl.TaskResult.Succeeded, "pip install was successful.");
+
+            } catch(err) {
+
+                return tl.setResult(tl.TaskResult.Failed, "pip install failed.");
             }
 
-            // Need error handling when selecting python for non Microsoft hosted agents
-            // Install junitparser
-            const pythonPath: string = tl.which('python3');
-            const python3: trm.ToolRunner = tl.tool(pythonPath);
-            python3.arg('-m');
-            python3.arg('pip');
-            python3.arg('install');
-            python3.arg('--upgrade');
-            python3.arg('junitparser');
-            const pipinstall: number = await python3.exec();
-            tl.setResult(tl.TaskResult.Succeeded, tl.loc('pipReturnCode', pipinstall));
-
-            // Generate the results
-            const genResults: trm.ToolRunner = tl.tool(pythonPath);
-            genResults.arg(path.join(__dirname, 'parsescaresults.py'));
-            genResults.arg('--target');
-            genResults.arg(`${appName}`);
-            genResults.arg('--mincvss');
-            genResults.arg(`${minCVSS}`);
-            genResults.arg('--failbuild');
-            genResults.arg(`${failBuild}`);
-            const publishResults: number = await genResults.exec();
-            tl.setResult(tl.TaskResult.Succeeded, tl.loc('bashReturnCode', publishResults));
-
-            return;
+            try {
+                // Generate the results
+                const genResults: trm.ToolRunner = tl.tool(pythonPath);
+                genResults.arg(path.join(__dirname, 'parsescaresults.py'));
+                genResults.arg('--target');
+                genResults.arg(`${appName}`);
+                genResults.arg('--mincvss');
+                genResults.arg(`${minCVSS}`);
+                genResults.arg('--failbuild');
+                genResults.arg(`${failBuild}`);
+                // Run the command
+                await genResults.exec();
+                return tl.setResult(tl.TaskResult.Succeeded, "SCA result parsing and upload was successful.");
+
+            } catch(err) {
+
+                return tl.setResult(tl.TaskResult.Failed, "SCA result parsing and upload failed.");
+            }
 
         } else {
             // Need to add Windows support
@@ -150,4 +125,60 @@ async function run(): Promise<void> {
     }
 }
 
+// Run the SCA scan
+async function runScan(scanType: string,
+                       scanTarget: string): Promise<void> {
+
+    try {
+        const curlPath: string = tl.which('curl', true);
+        const shPath: string = tl.which('sh', true);
+        const curl: trm.ToolRunner = tl.tool(curlPath);
+        curl.arg('-sSL');
+        curl.arg('https://download.sourceclear.com/ci.sh');
+        const sh: trm.ToolRunner = tl.tool(shPath);
+        sh.arg('-s');
+        sh.arg('--');
+        sh.arg('scan');
+        if (scanType !== 'directory') {
+            sh.arg(`--${scanType}`);
+        }
+        sh.arg(`${scanTarget}`);
+        sh.arg('--recursive');
+        sh.arg('--json');
+        sh.arg('scaresults.json');
+        const pipe: trm.ToolRunner = curl.pipeExecOutputToTool(sh);
+        await pipe.exec();
+
+        return tl.setResult(tl.TaskResult.Succeeded, "SCA scan completed.");
+
+    } catch (err) {
+        throw new Error(err);
+
+    }
+}
+
+// Test the SCA scan environment
+async function testSCA(): Promise<void> {
+
+    try {
+        const curlPath: string = tl.which('curl', true);
+        const shPath: string = tl.which('sh', true);
+        const curl: trm.ToolRunner = tl.tool(curlPath);
+        curl.arg('-sSL');
+        curl.arg('https://download.sourceclear.com/ci.sh');
+        const sh: trm.ToolRunner = tl.tool(shPath);
+        sh.arg('-s');
+        sh.arg('--');
+        sh.arg('test');
+        const pipe: trm.ToolRunner = curl.pipeExecOutputToTool(sh);
+        await pipe.exec();
+
+        return tl.setResult(tl.TaskResult.Succeeded, "SCA test completed.");
+
+    } catch (err) {
+        throw new Error(err);
+
+    }
+}
+
 run();

buildAndReleaseTask/task.json

@@ -89,11 +89,6 @@
         }
     },
     "showEnvironmentVariables": true,
-    "messages": {
-        "curlReturnCode": "Task exited with return code: %s",
-        "bashReturnCode": "Task exited with return code: %s",
-        "pipReturnCode": "Task exited with return code: %s"
-    },
     "visibility": [
         "Build",
         "Release"

requirements.txt

@@ -1,4 +1,4 @@
-junitparser==1.4.1
+junitparser==1.6.3
 pytest==6.1.1
 pytest-azurepipelines==0.8.0
 pytest-cov==2.10.1