Merge pull request #1516 from pulumi/tg/add-perf-test

add performance test including policy packs

misc/benchmarks/policy-pack/.gitignore

@@ -0,0 +1,2 @@
+/bin/
+/node_modules/

misc/benchmarks/policy-pack/PulumiPolicy.yaml

@@ -0,0 +1,2 @@
+runtime: nodejs
+description: A minimal Policy Pack for AWS using TypeScript.

misc/benchmarks/policy-pack/index.ts

@@ -0,0 +1,18 @@
+import * as aws from "@pulumi/aws";
+import { PolicyPack, validateResourceOfType } from "@pulumi/policy";
+
+new PolicyPack("aws-typescript", {
+    policies: [{
+        name: "s3-no-public-read",
+        description: "Prohibits setting the publicRead or publicReadWrite permission on AWS S3 buckets.",
+        enforcementLevel: "mandatory",
+        validateResource: validateResourceOfType(aws.s3.Bucket, (bucket, args, reportViolation) => {
+
+            if (bucket.acl === "public-read" || bucket.acl === "public-read-write") {
+                reportViolation(
+                    "You cannot set public-read or public-read-write on an S3 bucket. " +
+                    "Read more about ACLs here: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html");
+            }
+        }),
+    }],
+});

misc/benchmarks/policy-pack/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "aws-typescript",
+    "version": "0.0.1",
+    "dependencies": {
+        "@pulumi/aws": "^4.0.0",
+        "@pulumi/pulumi": "^3.0.0",
+        "@pulumi/policy": "^1.3.0"
+    },
+    "devDependencies": {
+        "@types/node": "^10.0.0"
+    }
+}

misc/benchmarks/policy-pack/tsconfig.json

@@ -0,0 +1,21 @@
+{
+    "compilerOptions": {
+        "outDir": "bin",
+        "target": "es6",
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "declaration": true,
+        "sourceMap": false,
+        "stripInternal": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true,
+        "strictNullChecks": true,
+    },
+    "files": [
+        "index.ts"
+    ]
+}

misc/test/performance_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
 	"path"
 	"testing"
 	"time"
@@ -115,6 +116,29 @@ func TestAccAwsPyS3Folder(t *testing.T) {
 	programTestAsBenchmark(t, benchmark, test)
 }
 
+func TestPolicyPacks(t *testing.T) {
+	policyPack := path.Join(getCwd(t), "..", "benchmarks", "policy-slow")
+
+	// Install the dependencies for the policy pack first
+	npmInstallCmd := exec.Command("npm", "install")
+	npmInstallCmd.Dir = policyPack
+	err := npmInstallCmd.Run()
+	assert.NoError(t, err)
+
+	benchmark := bench("aws-py-s3-folder", "aws", "go", "go")
+	opts := integration.ProgramTestOptions{
+		Dir:                    path.Join(getCwd(t), "..", "..", benchmark.Name),
+		UpdateCommandlineFlags: []string{fmt.Sprintf("--policy-pack=%s", policyPack)},
+		ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+			assertHTTPResult(t, "http://"+stack.Outputs["website_url"].(string), nil, func(body string) bool {
+				return assert.Contains(t, body, "Hello, Pulumi!")
+			})
+		},
+	}
+	test := getAWSBase(t).With(opts)
+	programTestAsBenchmark(t, benchmark, test)
+}
+
 type manyResourcesConfig struct {
 	folder       string
 	bench        traces.Benchmark
@@ -181,8 +205,8 @@ func TestManyResources(t *testing.T) {
 func programTestAsBenchmark(
 	t *testing.T,
 	bench traces.Benchmark,
-	test integration.ProgramTestOptions) {
-
+	test integration.ProgramTestOptions,
+) {
 	// Run preview only to make sure all needed plugins are
 	// downloaded so that these downloads do not skew
 	// measurements.