From 30cd85d8d60e82e0ecb849f3826d0ff6723ef8f1 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 27 Sep 2019 08:40:48 -0400
Subject: [PATCH 01/39] Fix typos

---
 README.md                                 | 2 +-
 schema/google/showcase/v1beta1/echo.proto | 6 +++---
 util/cmd/compile_protos/main.go           | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 7f87534a3..4f8df7b93 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,7 @@ hold using this installation method._
 
 ## Schema
 The schema of GAPIC Showcase API can be found in [schema/google/showcase/v1beta1](schema/google/showcase/v1beta1)
-It's dependencies can be found in the [googleapis/api-common-protos](https://github.com/googleapis/api-common-protos)
+Its dependencies can be found in the [googleapis/api-common-protos](https://github.com/googleapis/api-common-protos)
 submodule.
 
 ## Quick Start
diff --git a/schema/google/showcase/v1beta1/echo.proto b/schema/google/showcase/v1beta1/echo.proto
index 6e9d78a37..06b82b432 100644
--- a/schema/google/showcase/v1beta1/echo.proto
+++ b/schema/google/showcase/v1beta1/echo.proto
@@ -68,9 +68,9 @@ service Echo {
     };
   }
 
-  // This method, upon receiving a request on the stream, the same content will
-  // be passed  back on the stream. This method showcases bidirectional
-  // streaming rpcs.
+  // This method, upon receiving a request on the stream, will pass
+  // the same content back on the stream. This method showcases
+  // bidirectional streaming rpcs.
   rpc Chat(stream EchoRequest) returns (stream EchoResponse);
 
   // This is similar to the Expand method but instead of returning a stream of
diff --git a/util/cmd/compile_protos/main.go b/util/cmd/compile_protos/main.go
index a44579621..3308c8891 100644
--- a/util/cmd/compile_protos/main.go
+++ b/util/cmd/compile_protos/main.go
@@ -20,7 +20,7 @@ import (
 
 // This script regenerates all of the generated source code for the Showcase
 // API including the generated messages, gRPC services, go gapic clients,
-// and the generated CLI. This script must be ran from the root directory
+// and the generated CLI. This script must be run from the root directory
 // of the gapic-showcase repository.
 //
 // This script should be used whenever any changes are made to any of

From eaec5526d349613644892189c62707084e17ce24 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 27 Sep 2019 14:06:06 -0400
Subject: [PATCH 02/39] Bare-bones scaffold for acceptance harness

---
 cmd/qualify/qualify.go | 86 ++++++++++++++++++++++++++++++++++++++++++
 cmd/qualify/suite.go   | 60 +++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 cmd/qualify/qualify.go
 create mode 100644 cmd/qualify/suite.go

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
new file mode 100644
index 000000000..6aaace73d
--- /dev/null
+++ b/cmd/qualify/qualify.go
@@ -0,0 +1,86 @@
+package main
+
+import (
+	"log"
+	"os"
+)
+
+func main() {
+	const (
+		RetCodeSuccess = iota
+		RetCodeFailedDependencies
+		RetCodeUsageError
+		RetCodeInternalError
+		RetSuiteFailure
+	)
+
+	if err := checkDependencies(); err != nil {
+		os.Exit(RetCodeFailedDependencies)
+	}
+
+	showcasePID := startShowcase()
+	defer endProcess(showcasePID)
+
+	generatorName, viaProtoc, err := getGeneratorData()
+	if err != nil {
+		os.Exit(RetCodeUsageError)
+	}
+
+	allSuites := GetTestSuites(generatorName, viaProtoc)
+	success := true
+	for _, suite := range allSuites {
+		if err := suite.Run(); err != nil {
+			os.Exit(RetCodeInternalError)
+		}
+		if !suite.Success() {
+			success = false
+		}
+	}
+	if !success {
+		os.Exit(RetSuiteFailure)
+	}
+}
+
+func checkDependencies() error {
+	// TODO: add check for sample-tester
+	log.Printf("checkDependencies (TODO)")
+	return nil
+}
+
+// getGeneratorData obtains the name of the generaor from the command
+// line, and whether it is a protoc plugin (if not, it is considered
+// part of the monolith)
+func getGeneratorData() (string, bool, error) {
+	// TODO: Get from the command line
+	return "python", true, nil
+}
+
+func GetTestSuites(generatorName string, viaProtoc bool) []*Suite {
+	allSuites := []*Suite{}
+	defaultShowcasePort := 123 // TODO fix
+	// TODO: iterate over test suites
+	// TODO: get files in each suite
+	suiteFiles := []string{""}
+	newSuite := &Suite{
+		showcasePort: defaultShowcasePort,
+		viaProtoc:    viaProtoc,
+		generator:    generatorName,
+		files:        suiteFiles,
+	}
+	log.Printf("adding suite %#v", newSuite)
+	allSuites = append(allSuites, newSuite)
+	return allSuites
+}
+
+// startShowcase starts the Showcase server and returns its PID
+func startShowcase() int {
+	// TODO: fill in
+	log.Printf("startShowcase (TODO)")
+	return 0
+}
+
+// endProcess ends the process with the specified PID
+func endProcess(pid int) {
+	// TODO: fill in
+	log.Printf("endProcess (TODO): %v", pid)
+}
diff --git a/cmd/qualify/suite.go b/cmd/qualify/suite.go
new file mode 100644
index 000000000..f0b9bf91b
--- /dev/null
+++ b/cmd/qualify/suite.go
@@ -0,0 +1,60 @@
+package main
+
+import "log"
+
+type Suite struct {
+	name     string
+	location string
+	files    []string
+	sandbox  string
+
+	generator    string
+	showcasePort int
+	viaProtoc    bool
+}
+
+func (suite *Suite) Run() error {
+	if err := suite.Generate(); err != nil {
+		return err
+	}
+	suite.CheckGeneration()
+	suite.RunTests()
+	return nil
+}
+
+func (suite *Suite) Generate() error {
+	protos, gapic_yamls, sample_yamls := getGenerationFiles(suite.files)
+	suite.sandbox = createSandbox()
+
+	log.Printf("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
+	return nil
+}
+
+func (suite *Suite) CheckGeneration() {
+	// TODO: Fill in
+	log.Printf("CheckGeneration (TODO)")
+}
+
+func (suite *Suite) RunTests() {
+	// TODO: Fill in
+	log.Printf("RunTests (TODO)")
+}
+
+func (suite *Suite) Success() bool {
+	// TODO: fill in
+	return true
+}
+
+func getGenerationFiles(paths []string) (protos, gapic_yamls, sample_yamls []string) {
+	// TODO: extract all protos
+	// TODO: extract all gapic configs
+	// TODO: extract all sample_configs
+
+	log.Printf("getGenerationFiles (TODO)")
+	return nil, nil, nil
+}
+
+func createSandbox() string {
+	// TODO: make a temporary directory that can be deleted
+	return "/tmp"
+}

From 7b3e1007f8ccf709c0183d8ddd77456483bcfe70 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 27 Sep 2019 14:25:30 -0400
Subject: [PATCH 03/39] Move most logging to a debug logger

The debug logger will be switched on by CLI flags later; right now
it's hard-coded.
---
 cmd/qualify/qualify.go | 20 ++++++++++++++++----
 cmd/qualify/suite.go   | 13 +++++++------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 6aaace73d..e6ef9a25b 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -1,10 +1,14 @@
 package main
 
 import (
+	"io"
+	"io/ioutil"
 	"log"
 	"os"
 )
 
+var debugLog *log.Logger
+
 func main() {
 	const (
 		RetCodeSuccess = iota
@@ -14,6 +18,13 @@ func main() {
 		RetSuiteFailure
 	)
 
+	debugMe := true // TODO: get from CLI args
+	debugStream := io.Writer(os.Stderr)
+	if !debugMe {
+		debugStream = ioutil.Discard
+	}
+	debugLog = log.New(debugStream, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile)
+
 	if err := checkDependencies(); err != nil {
 		os.Exit(RetCodeFailedDependencies)
 	}
@@ -43,7 +54,7 @@ func main() {
 
 func checkDependencies() error {
 	// TODO: add check for sample-tester
-	log.Printf("checkDependencies (TODO)")
+	debugLog.Printf("checkDependencies (TODO)")
 	return nil
 }
 
@@ -66,8 +77,9 @@ func GetTestSuites(generatorName string, viaProtoc bool) []*Suite {
 		viaProtoc:    viaProtoc,
 		generator:    generatorName,
 		files:        suiteFiles,
+		debugLog:     debugLog,
 	}
-	log.Printf("adding suite %#v", newSuite)
+	debugLog.Printf("adding suite %#v", newSuite)
 	allSuites = append(allSuites, newSuite)
 	return allSuites
 }
@@ -75,12 +87,12 @@ func GetTestSuites(generatorName string, viaProtoc bool) []*Suite {
 // startShowcase starts the Showcase server and returns its PID
 func startShowcase() int {
 	// TODO: fill in
-	log.Printf("startShowcase (TODO)")
+	debugLog.Printf("startShowcase (TODO)")
 	return 0
 }
 
 // endProcess ends the process with the specified PID
 func endProcess(pid int) {
 	// TODO: fill in
-	log.Printf("endProcess (TODO): %v", pid)
+	debugLog.Printf("endProcess (TODO): %v", pid)
 }
diff --git a/cmd/qualify/suite.go b/cmd/qualify/suite.go
index f0b9bf91b..4c3a25615 100644
--- a/cmd/qualify/suite.go
+++ b/cmd/qualify/suite.go
@@ -7,6 +7,7 @@ type Suite struct {
 	location string
 	files    []string
 	sandbox  string
+	debugLog *log.Logger
 
 	generator    string
 	showcasePort int
@@ -23,21 +24,21 @@ func (suite *Suite) Run() error {
 }
 
 func (suite *Suite) Generate() error {
-	protos, gapic_yamls, sample_yamls := getGenerationFiles(suite.files)
+	protos, gapic_yamls, sample_yamls := suite.getGenerationFiles(suite.files)
 	suite.sandbox = createSandbox()
 
-	log.Printf("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
+	suite.debugLog.Printf("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
 	return nil
 }
 
 func (suite *Suite) CheckGeneration() {
 	// TODO: Fill in
-	log.Printf("CheckGeneration (TODO)")
+	suite.debugLog.Printf("CheckGeneration (TODO)")
 }
 
 func (suite *Suite) RunTests() {
 	// TODO: Fill in
-	log.Printf("RunTests (TODO)")
+	suite.debugLog.Printf("RunTests (TODO)")
 }
 
 func (suite *Suite) Success() bool {
@@ -45,12 +46,12 @@ func (suite *Suite) Success() bool {
 	return true
 }
 
-func getGenerationFiles(paths []string) (protos, gapic_yamls, sample_yamls []string) {
+func (suite *Suite) getGenerationFiles(paths []string) (protos, gapic_yamls, sample_yamls []string) {
 	// TODO: extract all protos
 	// TODO: extract all gapic configs
 	// TODO: extract all sample_configs
 
-	log.Printf("getGenerationFiles (TODO)")
+	suite.debugLog.Printf("getGenerationFiles (TODO)")
 	return nil, nil, nil
 }
 

From 93121583aae8e64596350756e0edcb0db98f5b7e Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 27 Sep 2019 15:09:00 -0400
Subject: [PATCH 04/39] Iterate over all suite config dirs. Add a simple suite
 config.

---
 acceptance/passing/echo.proto |  1 +
 cmd/qualify/qualify.go        | 80 ++++++++++++++++++++++++++++-------
 2 files changed, 66 insertions(+), 15 deletions(-)
 create mode 120000 acceptance/passing/echo.proto

diff --git a/acceptance/passing/echo.proto b/acceptance/passing/echo.proto
new file mode 120000
index 000000000..a63c95653
--- /dev/null
+++ b/acceptance/passing/echo.proto
@@ -0,0 +1 @@
+../../schema/google/showcase/v1beta1/echo.proto
\ No newline at end of file
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index e6ef9a25b..82d1efaab 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -5,6 +5,8 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
+	"path/filepath"
 )
 
 var debugLog *log.Logger
@@ -37,7 +39,11 @@ func main() {
 		os.Exit(RetCodeUsageError)
 	}
 
-	allSuites := GetTestSuites(generatorName, viaProtoc)
+	allSuites, err := GetTestSuites(generatorName, viaProtoc)
+	if err != nil {
+		os.Exit(RetCodeInternalError)
+	}
+
 	success := true
 	for _, suite := range allSuites {
 		if err := suite.Run(); err != nil {
@@ -66,22 +72,47 @@ func getGeneratorData() (string, bool, error) {
 	return "python", true, nil
 }
 
-func GetTestSuites(generatorName string, viaProtoc bool) []*Suite {
+// GetTestSuites returns a list of Suite as found in the specified
+// suite root directory.
+func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
+	suiteRootPath := "../../acceptance"
+
+	debugLog.Printf("GetTestSuites: generator=%q   protoc-%v", generatorName, viaProtoc)
 	allSuites := []*Suite{}
-	defaultShowcasePort := 123 // TODO fix
-	// TODO: iterate over test suites
-	// TODO: get files in each suite
-	suiteFiles := []string{""}
-	newSuite := &Suite{
-		showcasePort: defaultShowcasePort,
-		viaProtoc:    viaProtoc,
-		generator:    generatorName,
-		files:        suiteFiles,
-		debugLog:     debugLog,
+	suiteEntries, err := ioutil.ReadDir(suiteRootPath)
+	if err != nil {
+		return nil, err
 	}
-	debugLog.Printf("adding suite %#v", newSuite)
-	allSuites = append(allSuites, newSuite)
-	return allSuites
+
+	for _, suiteDir := range suiteEntries {
+		if !suiteDir.IsDir() {
+			continue
+		}
+		name := suiteDir.Name()
+		location, err := filepath.Abs(path.Join(suiteRootPath, name))
+		if err != nil {
+			return nil, err
+		}
+
+		suiteFiles, err := getAllFiles(location)
+		if err != nil {
+			return nil, err
+		}
+
+		defaultShowcasePort := 123 // TODO fix
+		newSuite := &Suite{
+			name:         name,
+			location:     location,
+			showcasePort: defaultShowcasePort,
+			viaProtoc:    viaProtoc,
+			generator:    generatorName,
+			files:        suiteFiles,
+			debugLog:     debugLog,
+		}
+		debugLog.Printf("adding suite %#v", newSuite)
+		allSuites = append(allSuites, newSuite)
+	}
+	return allSuites, nil
 }
 
 // startShowcase starts the Showcase server and returns its PID
@@ -96,3 +127,22 @@ func endProcess(pid int) {
 	// TODO: fill in
 	debugLog.Printf("endProcess (TODO): %v", pid)
 }
+
+// getAllFiles returns a list of all the files (excluding directories)
+// at any level under `root`.
+func getAllFiles(root string) ([]string, error) {
+	debugLog.Printf("getAllFiles: root=%q", root)
+	allFiles := []string{}
+	err := filepath.Walk(root,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			allFiles = append(allFiles, path)
+			return nil
+		})
+	return allFiles, err
+}

From 1ba8c2c8f8b2188ae6f89a8f28ae7e23d9323efe Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 28 Sep 2019 11:56:04 -0400
Subject: [PATCH 05/39] Check needed dependencies. Add script to prepare them.

---
 cmd/qualify/prepare-to-qualify | 36 ++++++++++++++++++++++++++++++++++
 cmd/qualify/qualify.go         | 34 +++++++++++++++++++++++++++++++-
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 cmd/qualify/prepare-to-qualify

diff --git a/cmd/qualify/prepare-to-qualify b/cmd/qualify/prepare-to-qualify
new file mode 100644
index 000000000..dbe39e20d
--- /dev/null
+++ b/cmd/qualify/prepare-to-qualify
@@ -0,0 +1,36 @@
+#!/bin/bash
+VENV_NAME=venv/qualify
+DATE=$(date +'%Y-%m-%d.%H-%M')
+
+verify() {
+  name=$1
+  $name --version  >& /dev/null || { echo "Please install $name to continue" ; return 2 ; }  
+}
+
+verify python3
+verify pip
+verify curl
+
+INSTALL_DIR=${1:-$(mktemp -d -t qualify.${DATE}.XXXXXX)}
+pushd ${INSTALL_DIR} >& /dev/null
+
+
+VENV_CREATE="python3 -m venv $VENV_NAME"
+SHOWCASE_SEMVER=0.1.0
+HOST_OS=$(go env GOHOSTOS)
+HOST_ARCH=$(go env GOHOSTARCH)
+
+echo '=== creating venv'  && \
+  ${VENV_CREATE} || { python3 -m pip install virtualenv && ${VENV_ACTIVATE} ; } && \
+  echo "=== activating venv"  && \
+  . ${VENV_NAME}/bin/activate && \
+  echo "=== installing pip" && \
+  pip install sample-tester && \
+  echo "=== installing showcase" && \
+  curl -sSL  https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_SEMVER}/gapic-showcase-${SHOWCASE_SEMVER}-${HOST_OS}-${HOST_ARCH}.tar.gz | tar xz &&
+  export PATH=$(pwd):${PATH}
+
+
+popd >& /dev/null
+echo -e "\nInstalled in: ${INSTALL_DIR}"
+
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 82d1efaab..231ec116b 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -1,14 +1,25 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 )
 
+/* Development instructions
+
+. ./prepare-t-qualify
+
+=== Sample run ======
+go run qualify.go suite.go
+
+*/
+
 var debugLog *log.Logger
 
 func main() {
@@ -60,7 +71,28 @@ func main() {
 
 func checkDependencies() error {
 	// TODO: add check for sample-tester
-	debugLog.Printf("checkDependencies (TODO)")
+	showcaseCmd := "gapic-showcase"
+	sampleTesterCmd := "sample-tester"
+	notFound := []string{}
+	debugLog.Printf("checkDependencies")
+
+	showcasePath, err := exec.LookPath(showcaseCmd)
+	if err != nil {
+		notFound = append(notFound, showcaseCmd)
+	}
+
+	sampleTesterPath, err := exec.LookPath(sampleTesterCmd)
+	if err != nil {
+		notFound = append(notFound, sampleTesterCmd)
+	}
+
+	if len(notFound) > 0 {
+		msg := fmt.Sprintf("could not find dependencies in $PATH: %q", notFound)
+		log.Printf(msg)
+		return fmt.Errorf(msg)
+	}
+	debugLog.Printf("found %q: %s", showcaseCmd, showcasePath)
+	debugLog.Printf("found %q: %s", sampleTesterCmd, sampleTesterPath)
 	return nil
 }
 

From f0abe48813e24e9a22db0d6d57a598cae1278308 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 28 Sep 2019 12:23:43 -0400
Subject: [PATCH 06/39] Start and stop gapic-showcase in the background.

---
 cmd/qualify/qualify.go | 46 +++++++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 231ec116b..d9e4b49ef 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -22,12 +22,15 @@ go run qualify.go suite.go
 
 var debugLog *log.Logger
 
+const showcaseCmd = "gapic-showcase"
+
 func main() {
 	const (
 		RetCodeSuccess = iota
+		RetCodeInternalError
 		RetCodeFailedDependencies
 		RetCodeUsageError
-		RetCodeInternalError
+		RetCodeCantRunShowcase
 		RetSuiteFailure
 	)
 
@@ -42,8 +45,11 @@ func main() {
 		os.Exit(RetCodeFailedDependencies)
 	}
 
-	showcasePID := startShowcase()
-	defer endProcess(showcasePID)
+	showcase, err := startShowcase()
+	if err != nil {
+		os.Exit(RetCodeCantRunShowcase)
+	}
+	defer endProcess(showcase)
 
 	generatorName, viaProtoc, err := getGeneratorData()
 	if err != nil {
@@ -71,10 +77,9 @@ func main() {
 
 func checkDependencies() error {
 	// TODO: add check for sample-tester
-	showcaseCmd := "gapic-showcase"
 	sampleTesterCmd := "sample-tester"
 	notFound := []string{}
-	debugLog.Printf("checkDependencies")
+	debugLog.Printf("checkDependencies()")
 
 	showcasePath, err := exec.LookPath(showcaseCmd)
 	if err != nil {
@@ -109,7 +114,7 @@ func getGeneratorData() (string, bool, error) {
 func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 	suiteRootPath := "../../acceptance"
 
-	debugLog.Printf("GetTestSuites: generator=%q   protoc-%v", generatorName, viaProtoc)
+	debugLog.Printf("GetTestSuites(generator=%q, protoc=%v)", generatorName, viaProtoc)
 	allSuites := []*Suite{}
 	suiteEntries, err := ioutil.ReadDir(suiteRootPath)
 	if err != nil {
@@ -148,22 +153,39 @@ func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 }
 
 // startShowcase starts the Showcase server and returns its PID
-func startShowcase() int {
+func startShowcase() (*exec.Cmd, error) {
 	// TODO: fill in
-	debugLog.Printf("startShowcase (TODO)")
-	return 0
+	debugLog.Printf("startShowcase()")
+	cmd := exec.Command(showcaseCmd)
+	err := cmd.Start()
+	return cmd, err
 }
 
 // endProcess ends the process with the specified PID
-func endProcess(pid int) {
+func endProcess(cmd *exec.Cmd) error {
 	// TODO: fill in
-	debugLog.Printf("endProcess (TODO): %v", pid)
+	debugLog.Printf("endProcess(%v)", cmd)
+	process := cmd.Process
+	if err := process.Signal(os.Interrupt); err != nil {
+		msg := fmt.Sprintf("could not end process %v normally", process.Pid)
+		if err := process.Kill(); err != nil {
+			msg = fmt.Sprintf("%s; killing failed. Please kill manually!", msg)
+			log.Printf(msg)
+			return fmt.Errorf(msg)
+		}
+		msg = fmt.Sprintf("%s but killing it succeeded", msg)
+		log.Printf(msg)
+	}
+	debugLog.Printf("waiting for process to end: %v (%v)", process.Pid, cmd.Path)
+	cmd.Wait()
+	debugLog.Printf("process ended:              %v (%v) ended", process.Pid, cmd.Path)
+	return nil
 }
 
 // getAllFiles returns a list of all the files (excluding directories)
 // at any level under `root`.
 func getAllFiles(root string) ([]string, error) {
-	debugLog.Printf("getAllFiles: root=%q", root)
+	debugLog.Printf("getAllFiles(root=%q)", root)
 	allFiles := []string{}
 	err := filepath.Walk(root,
 		func(path string, info os.FileInfo, err error) error {

From a65893d21326a6cb767494c60a0833768b72202d Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 28 Sep 2019 14:05:21 -0400
Subject: [PATCH 07/39] Build gapic-showcase from local tree.

Instead of fetching a relese. This is for the development
prepare-to-qualify script.
---
 cmd/qualify/prepare-to-qualify | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmd/qualify/prepare-to-qualify b/cmd/qualify/prepare-to-qualify
index dbe39e20d..09f0e6fa0 100644
--- a/cmd/qualify/prepare-to-qualify
+++ b/cmd/qualify/prepare-to-qualify
@@ -2,6 +2,9 @@
 VENV_NAME=venv/qualify
 DATE=$(date +'%Y-%m-%d.%H-%M')
 
+BUILD_DIR=$(pwd)
+go build ../gapic-showcase/
+
 verify() {
   name=$1
   $name --version  >& /dev/null || { echo "Please install $name to continue" ; return 2 ; }  
@@ -16,9 +19,6 @@ pushd ${INSTALL_DIR} >& /dev/null
 
 
 VENV_CREATE="python3 -m venv $VENV_NAME"
-SHOWCASE_SEMVER=0.1.0
-HOST_OS=$(go env GOHOSTOS)
-HOST_ARCH=$(go env GOHOSTARCH)
 
 echo '=== creating venv'  && \
   ${VENV_CREATE} || { python3 -m pip install virtualenv && ${VENV_ACTIVATE} ; } && \
@@ -27,7 +27,7 @@ echo '=== creating venv'  && \
   echo "=== installing pip" && \
   pip install sample-tester && \
   echo "=== installing showcase" && \
-  curl -sSL  https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_SEMVER}/gapic-showcase-${SHOWCASE_SEMVER}-${HOST_OS}-${HOST_ARCH}.tar.gz | tar xz &&
+  mv ${BUILD_DIR}/gapic-showcase . && \
   export PATH=$(pwd):${PATH}
 
 

From 4fac512968d2e1879d9679d640d8b4f936df669c Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 30 Sep 2019 16:47:43 -0700
Subject: [PATCH 08/39] Add some comments/TODos

---
 cmd/qualify/qualify.go | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index d9e4b49ef..caedc5662 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -13,16 +13,21 @@ import (
 
 /* Development instructions
 
-. ./prepare-t-qualify
+. ./prepare-to-qualify
 
 === Sample run ======
 go run qualify.go suite.go
 
-*/
+Notes for upcoming development:
+packing assets into Go files:
+https://www.google.com/search?q=golang+ship+data+files+with+binary&oq=golang+ship+data+files+with+binary
+https://github.com/go-bindata/go-bindata
+https://github.com/gobuffalo/packr
+Choosing: https://tech.townsourced.com/post/embedding-static-files-in-go/
 
 var debugLog *log.Logger
 
-const showcaseCmd = "gapic-showcase"
+const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
 func main() {
 	const (

From 54ddeb5cf56cdd7b2e00da4baa56902262a73936 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 30 Sep 2019 16:48:32 -0700
Subject: [PATCH 09/39] use go-trace package for debug statements

---
 cmd/qualify/qualify.go | 32 +++++++++++++++-----------------
 cmd/qualify/suite.go   | 13 +++++++------
 go.mod                 |  1 +
 go.sum                 |  2 ++
 4 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index caedc5662..a8741a5fc 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -2,13 +2,14 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+
+	trace "github.com/google/go-trace"
 )
 
 /* Development instructions
@@ -25,7 +26,7 @@ https://github.com/go-bindata/go-bindata
 https://github.com/gobuffalo/packr
 Choosing: https://tech.townsourced.com/post/embedding-static-files-in-go/
 
-var debugLog *log.Logger
+*/
 
 const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
@@ -40,11 +41,9 @@ func main() {
 	)
 
 	debugMe := true // TODO: get from CLI args
-	debugStream := io.Writer(os.Stderr)
-	if !debugMe {
-		debugStream = ioutil.Discard
+	if debugMe {
+		trace.On(true)
 	}
-	debugLog = log.New(debugStream, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile)
 
 	if err := checkDependencies(); err != nil {
 		os.Exit(RetCodeFailedDependencies)
@@ -84,7 +83,7 @@ func checkDependencies() error {
 	// TODO: add check for sample-tester
 	sampleTesterCmd := "sample-tester"
 	notFound := []string{}
-	debugLog.Printf("checkDependencies()")
+	trace.Trace("")
 
 	showcasePath, err := exec.LookPath(showcaseCmd)
 	if err != nil {
@@ -101,8 +100,8 @@ func checkDependencies() error {
 		log.Printf(msg)
 		return fmt.Errorf(msg)
 	}
-	debugLog.Printf("found %q: %s", showcaseCmd, showcasePath)
-	debugLog.Printf("found %q: %s", sampleTesterCmd, sampleTesterPath)
+	trace.Trace("found %q: %s", showcaseCmd, showcasePath)
+	trace.Trace("found %q: %s", sampleTesterCmd, sampleTesterPath)
 	return nil
 }
 
@@ -119,7 +118,7 @@ func getGeneratorData() (string, bool, error) {
 func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 	suiteRootPath := "../../acceptance"
 
-	debugLog.Printf("GetTestSuites(generator=%q, protoc=%v)", generatorName, viaProtoc)
+	trace.Trace("(generator=%q, protoc=%v)", generatorName, viaProtoc)
 	allSuites := []*Suite{}
 	suiteEntries, err := ioutil.ReadDir(suiteRootPath)
 	if err != nil {
@@ -149,9 +148,8 @@ func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 			viaProtoc:    viaProtoc,
 			generator:    generatorName,
 			files:        suiteFiles,
-			debugLog:     debugLog,
 		}
-		debugLog.Printf("adding suite %#v", newSuite)
+		trace.Trace("adding suite %#v", newSuite)
 		allSuites = append(allSuites, newSuite)
 	}
 	return allSuites, nil
@@ -160,7 +158,7 @@ func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 // startShowcase starts the Showcase server and returns its PID
 func startShowcase() (*exec.Cmd, error) {
 	// TODO: fill in
-	debugLog.Printf("startShowcase()")
+	trace.Trace("")
 	cmd := exec.Command(showcaseCmd)
 	err := cmd.Start()
 	return cmd, err
@@ -169,7 +167,7 @@ func startShowcase() (*exec.Cmd, error) {
 // endProcess ends the process with the specified PID
 func endProcess(cmd *exec.Cmd) error {
 	// TODO: fill in
-	debugLog.Printf("endProcess(%v)", cmd)
+	trace.Trace("cmd=%v)", cmd)
 	process := cmd.Process
 	if err := process.Signal(os.Interrupt); err != nil {
 		msg := fmt.Sprintf("could not end process %v normally", process.Pid)
@@ -181,16 +179,16 @@ func endProcess(cmd *exec.Cmd) error {
 		msg = fmt.Sprintf("%s but killing it succeeded", msg)
 		log.Printf(msg)
 	}
-	debugLog.Printf("waiting for process to end: %v (%v)", process.Pid, cmd.Path)
+	trace.Trace("waiting for process to end: %v (%v)", process.Pid, cmd.Path)
 	cmd.Wait()
-	debugLog.Printf("process ended:              %v (%v) ended", process.Pid, cmd.Path)
+	trace.Trace("process ended:              %v (%v) ended", process.Pid, cmd.Path)
 	return nil
 }
 
 // getAllFiles returns a list of all the files (excluding directories)
 // at any level under `root`.
 func getAllFiles(root string) ([]string, error) {
-	debugLog.Printf("getAllFiles(root=%q)", root)
+	trace.Trace("root=%q", root)
 	allFiles := []string{}
 	err := filepath.Walk(root,
 		func(path string, info os.FileInfo, err error) error {
diff --git a/cmd/qualify/suite.go b/cmd/qualify/suite.go
index 4c3a25615..a2f6c627c 100644
--- a/cmd/qualify/suite.go
+++ b/cmd/qualify/suite.go
@@ -1,13 +1,14 @@
 package main
 
-import "log"
+import (
+	trace "github.com/google/go-trace"
+)
 
 type Suite struct {
 	name     string
 	location string
 	files    []string
 	sandbox  string
-	debugLog *log.Logger
 
 	generator    string
 	showcasePort int
@@ -27,18 +28,18 @@ func (suite *Suite) Generate() error {
 	protos, gapic_yamls, sample_yamls := suite.getGenerationFiles(suite.files)
 	suite.sandbox = createSandbox()
 
-	suite.debugLog.Printf("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
+	trace.Trace("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
 	return nil
 }
 
 func (suite *Suite) CheckGeneration() {
 	// TODO: Fill in
-	suite.debugLog.Printf("CheckGeneration (TODO)")
+	trace.Trace("CheckGeneration (TODO)")
 }
 
 func (suite *Suite) RunTests() {
 	// TODO: Fill in
-	suite.debugLog.Printf("RunTests (TODO)")
+	trace.Trace("RunTests (TODO)")
 }
 
 func (suite *Suite) Success() bool {
@@ -51,7 +52,7 @@ func (suite *Suite) getGenerationFiles(paths []string) (protos, gapic_yamls, sam
 	// TODO: extract all gapic configs
 	// TODO: extract all sample_configs
 
-	suite.debugLog.Printf("getGenerationFiles (TODO)")
+	trace.Trace("getGenerationFiles (TODO)")
 	return nil, nil, nil
 }
 
diff --git a/go.mod b/go.mod
index 077d74964..e4b2fecc5 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/googleapis/gapic-showcase
 require (
 	cloud.google.com/go v0.47.0
 	github.com/golang/protobuf v1.3.2
+	github.com/google/go-trace v0.0.0-20180203010202-d282251f7009
 	github.com/googleapis/gax-go/v2 v2.0.5
 	github.com/googleapis/grpc-fallback-go v0.1.3
 	github.com/spf13/cobra v0.0.5
diff --git a/go.sum b/go.sum
index 126777816..d5dcab163 100644
--- a/go.sum
+++ b/go.sum
@@ -64,6 +64,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-trace v0.0.0-20180203010202-d282251f7009 h1:fzuC1Phituau4YOSDoX9ZPi9DnnuReeXoONKMOMDojI=
+github.com/google/go-trace v0.0.0-20180203010202-d282251f7009/go.mod h1:Z8aRH4v1BFRrLDTZF2jr96KpzD7RHBRMAzVnnKhgKzM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=

From 11396171b1ada42c478d01616da4f5a403610793 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Tue, 1 Oct 2019 19:31:51 -0700
Subject: [PATCH 10/39] Classify suite files by type

---
 cmd/qualify/prepare-to-qualify | 13 +++---
 cmd/qualify/qualify.go         |  3 +-
 cmd/qualify/suite.go           | 76 ++++++++++++++++++++++++++++------
 go.mod                         |  1 +
 4 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/cmd/qualify/prepare-to-qualify b/cmd/qualify/prepare-to-qualify
index 09f0e6fa0..111b5fd8f 100644
--- a/cmd/qualify/prepare-to-qualify
+++ b/cmd/qualify/prepare-to-qualify
@@ -10,12 +10,13 @@ verify() {
   $name --version  >& /dev/null || { echo "Please install $name to continue" ; return 2 ; }  
 }
 
-verify python3
-verify pip
-verify curl
+verify python3 || return $?
+# verify protoc || return $?
+verify pip || return $?
+verify curl || return $?
 
-INSTALL_DIR=${1:-$(mktemp -d -t qualify.${DATE}.XXXXXX)}
-pushd ${INSTALL_DIR} >& /dev/null
+export GAPIC_QUALIFY_DIR=${1:-$(mktemp -d -t qualify.${DATE}.XXXXXX)}
+pushd ${GAPIC_QUALIFY_DIR} >& /dev/null
 
 
 VENV_CREATE="python3 -m venv $VENV_NAME"
@@ -32,5 +33,5 @@ echo '=== creating venv'  && \
 
 
 popd >& /dev/null
-echo -e "\nInstalled in: ${INSTALL_DIR}"
+echo -e "\nInstalled in: ${GAPIC_QUALIFY_DIR}"
 
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index a8741a5fc..26a9678de 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -80,7 +80,6 @@ func main() {
 }
 
 func checkDependencies() error {
-	// TODO: add check for sample-tester
 	sampleTesterCmd := "sample-tester"
 	notFound := []string{}
 	trace.Trace("")
@@ -105,7 +104,7 @@ func checkDependencies() error {
 	return nil
 }
 
-// getGeneratorData obtains the name of the generaor from the command
+// getGeneratorData obtains the name of the generator from the command
 // line, and whether it is a protoc plugin (if not, it is considered
 // part of the monolith)
 func getGeneratorData() (string, bool, error) {
diff --git a/cmd/qualify/suite.go b/cmd/qualify/suite.go
index a2f6c627c..2c3ea9109 100644
--- a/cmd/qualify/suite.go
+++ b/cmd/qualify/suite.go
@@ -1,14 +1,21 @@
 package main
 
 import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+
 	trace "github.com/google/go-trace"
+	yaml "gopkg.in/yaml.v2"
 )
 
 type Suite struct {
-	name     string
-	location string
-	files    []string
-	sandbox  string
+	name        string
+	location    string
+	files       []string
+	sandbox     string
+	filesByType map[string][]string
 
 	generator    string
 	showcasePort int
@@ -25,10 +32,12 @@ func (suite *Suite) Run() error {
 }
 
 func (suite *Suite) Generate() error {
-	protos, gapic_yamls, sample_yamls := suite.getGenerationFiles(suite.files)
+	if err := suite.getGenerationFiles(); err != nil {
+		return err
+	}
 	suite.sandbox = createSandbox()
 
-	trace.Trace("Generate (TODO): protos: %v    gapic_yamls: %v    sample_yamls: %v", protos, gapic_yamls, sample_yamls)
+	trace.Trace("Generate (TODO)")
 	return nil
 }
 
@@ -47,13 +56,56 @@ func (suite *Suite) Success() bool {
 	return true
 }
 
-func (suite *Suite) getGenerationFiles(paths []string) (protos, gapic_yamls, sample_yamls []string) {
-	// TODO: extract all protos
-	// TODO: extract all gapic configs
-	// TODO: extract all sample_configs
+func (suite *Suite) getGenerationFiles() (err error) {
+	suite.filesByType = make(map[string][]string)
+	for _, thisFile := range suite.files {
+		extension := filepath.Ext(thisFile)
+		var fileTypes []string
+		switch extension {
 
-	trace.Trace("getGenerationFiles (TODO)")
-	return nil, nil, nil
+		case ".proto":
+			fileTypes = []string{"proto"}
+		case ".yaml", ".yml":
+			trace.Trace("reading %s", thisFile)
+			content, err := ioutil.ReadFile(thisFile)
+			if err != nil {
+				trace.Trace("error reading: %s", err)
+				return err
+			}
+			fileTypes, err = yamlDocTypes(content)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, oneType := range fileTypes {
+			similarFiles := suite.filesByType[oneType]
+			suite.filesByType[oneType] = append(similarFiles, thisFile)
+			trace.Trace("%s: type %q", thisFile, oneType)
+		}
+	}
+	return nil
+}
+
+func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
+	type docSchema struct {
+		Type string
+	}
+
+	decoder := yaml.NewDecoder(bytes.NewReader(fileContent))
+	for {
+		doc := &docSchema{Type: "(UNKNOWN)"}
+		err = decoder.Decode(doc)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			trace.Trace("error decoding: %s", err)
+			return
+		}
+		docTypes = append(docTypes, doc.Type)
+	}
+	return docTypes, nil
 }
 
 func createSandbox() string {
diff --git a/go.mod b/go.mod
index e4b2fecc5..c11e1338f 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
 	google.golang.org/api v0.11.0
 	google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03
 	google.golang.org/grpc v1.24.0
+	gopkg.in/yaml.v2 v2.2.2
 )
 
 go 1.13

From 8a339c3427fb528f66db7de17ed36fb4a6abcead Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Wed, 2 Oct 2019 17:32:40 -0700
Subject: [PATCH 11/39] Add instructions for running protoc w Go plugin

Capture the language name and options in preparation for running the
protoc plugin from this acceptance harness
---
 acceptance/passing/google      |  1 +
 acceptance/passing/sample.yaml | 16 ++++++++++++++++
 cmd/qualify/generator.go       |  8 ++++++++
 cmd/qualify/prepare-to-qualify | 12 ++++++++++--
 cmd/qualify/qualify.go         | 27 +++++++++++++++++----------
 cmd/qualify/suite.go           |  3 +--
 util/cmd/release/main.go       |  4 ++--
 7 files changed, 55 insertions(+), 16 deletions(-)
 create mode 120000 acceptance/passing/google
 create mode 100644 acceptance/passing/sample.yaml
 create mode 100644 cmd/qualify/generator.go

diff --git a/acceptance/passing/google b/acceptance/passing/google
new file mode 120000
index 000000000..82d2520f7
--- /dev/null
+++ b/acceptance/passing/google
@@ -0,0 +1 @@
+../../schema/api-common-protos/google
\ No newline at end of file
diff --git a/acceptance/passing/sample.yaml b/acceptance/passing/sample.yaml
new file mode 100644
index 000000000..c4a2e3ad0
--- /dev/null
+++ b/acceptance/passing/sample.yaml
@@ -0,0 +1,16 @@
+---
+type: sampleconfig
+schema_version: 2
+---
+type: test/sample
+schema_version: 1
+---
+type: inventory
+widget: "$expensive"
+---
+elements:
+  - hydrogen
+  - oxygen
+---
+type: sample/other
+tool: my_yaml
diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
new file mode 100644
index 000000000..cc3b74fef
--- /dev/null
+++ b/cmd/qualify/generator.go
@@ -0,0 +1,8 @@
+package main
+
+type GeneratorInfo struct {
+	name       string
+	dir        string
+	options    string
+	isMonolith bool
+}
diff --git a/cmd/qualify/prepare-to-qualify b/cmd/qualify/prepare-to-qualify
index 111b5fd8f..53842a18c 100644
--- a/cmd/qualify/prepare-to-qualify
+++ b/cmd/qualify/prepare-to-qualify
@@ -7,11 +7,12 @@ go build ../gapic-showcase/
 
 verify() {
   name=$1
-  $name --version  >& /dev/null || { echo "Please install $name to continue" ; return 2 ; }  
+  msg=$2
+  $name --version  >& /dev/null || { echo "Please install $name to continue. $2" ; return 2 ; }  
 }
 
 verify python3 || return $?
-# verify protoc || return $?
+verify protoc "See https://github.com/protocolbuffers/protobuf/releases" || return $?
 verify pip || return $?
 verify curl || return $?
 
@@ -35,3 +36,10 @@ echo '=== creating venv'  && \
 popd >& /dev/null
 echo -e "\nInstalled in: ${GAPIC_QUALIFY_DIR}"
 
+# To get the Go plugin to work:
+#
+# VC_PROTO_PATH=/home/vchudnov/Work/ACTools/gapic-showcase/acceptance/passing
+# GO_PLUGIN_DIR=/tmp/goinstall
+# mkdir ${GO_PLUGIN_DIR}
+# GOPATH=${GO_PLUGIN_DIR} go install github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
+# protoc --go_gapic_out ./gen --go_gapic_opt 'go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase' --plugin=${GO_PLUGIN_DIR}/bin/protoc-gen-go_gapic --proto_path=${VC_PROTO_PATH} ${VC_PROTO_PATH}/echo.proto
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 26a9678de..fd8635665 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -17,7 +17,7 @@ import (
 . ./prepare-to-qualify
 
 === Sample run ======
-go run qualify.go suite.go
+go run *.go
 
 Notes for upcoming development:
 packing assets into Go files:
@@ -55,12 +55,12 @@ func main() {
 	}
 	defer endProcess(showcase)
 
-	generatorName, viaProtoc, err := getGeneratorData()
+	generatorData, err := getGeneratorData()
 	if err != nil {
 		os.Exit(RetCodeUsageError)
 	}
 
-	allSuites, err := GetTestSuites(generatorName, viaProtoc)
+	allSuites, err := GetTestSuites(generatorData)
 	if err != nil {
 		os.Exit(RetCodeInternalError)
 	}
@@ -107,17 +107,25 @@ func checkDependencies() error {
 // getGeneratorData obtains the name of the generator from the command
 // line, and whether it is a protoc plugin (if not, it is considered
 // part of the monolith)
-func getGeneratorData() (string, bool, error) {
-	// TODO: Get from the command line
-	return "python", true, nil
+func getGeneratorData() (*GeneratorInfo, error) {
+	pluginName := "go"                                                                 // TODO: get from CLI args
+	pluginDir := "/tmp/goinstall/bin"                                                  // TODO: get from CLI args
+	pluginOpt := "'go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase'" // TODO: get from CLI args
+	generator := &GeneratorInfo{
+		name:    pluginName,
+		dir:     pluginDir,
+		options: pluginOpt,
+	}
+	trace.Trace("hard-coded generator=%#v", generator)
+	return generator, nil
 }
 
 // GetTestSuites returns a list of Suite as found in the specified
 // suite root directory.
-func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
+func GetTestSuites(generator *GeneratorInfo) ([]*Suite, error) {
 	suiteRootPath := "../../acceptance"
 
-	trace.Trace("(generator=%q, protoc=%v)", generatorName, viaProtoc)
+	trace.Trace("(generator=%#v)", generator)
 	allSuites := []*Suite{}
 	suiteEntries, err := ioutil.ReadDir(suiteRootPath)
 	if err != nil {
@@ -144,8 +152,7 @@ func GetTestSuites(generatorName string, viaProtoc bool) ([]*Suite, error) {
 			name:         name,
 			location:     location,
 			showcasePort: defaultShowcasePort,
-			viaProtoc:    viaProtoc,
-			generator:    generatorName,
+			generator:    generator,
 			files:        suiteFiles,
 		}
 		trace.Trace("adding suite %#v", newSuite)
diff --git a/cmd/qualify/suite.go b/cmd/qualify/suite.go
index 2c3ea9109..b96d2431e 100644
--- a/cmd/qualify/suite.go
+++ b/cmd/qualify/suite.go
@@ -17,9 +17,8 @@ type Suite struct {
 	sandbox     string
 	filesByType map[string][]string
 
-	generator    string
+	generator    *GeneratorInfo
 	showcasePort int
-	viaProtoc    bool
 }
 
 func (suite *Suite) Run() error {
diff --git a/util/cmd/release/main.go b/util/cmd/release/main.go
index 3ff99a617..84a659469 100644
--- a/util/cmd/release/main.go
+++ b/util/cmd/release/main.go
@@ -25,11 +25,11 @@ import (
 )
 
 // This script is ran in CI when a new version tag is pushed to master. This script
-// places the compiled proto descriptor set, a tarball of showcase-protos alongside it's
+// places the compiled proto descriptor set, a tarball of showcase-protos alongside its
 // dependencies, and the compiled executables of the gapic-showcase cli tool inside the
 // directory "dist"
 //
-// This script must be ran from the root directory of the gapic-showcase repository.
+// This script must be run from the root directory of the gapic-showcase repository.
 //
 // Usage: go run ./util/cmd/release
 func main() {

From d09898f9634143289989cd2bdfddd67899e7318f Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Thu, 3 Oct 2019 16:05:05 -0700
Subject: [PATCH 12/39] Use packr to get files from disk

---
 cmd/qualify/assets.go  | 53 ++++++++++++++++++++++++++++++++++++++++++
 cmd/qualify/qualify.go | 40 +++++++------------------------
 go.sum                 | 29 +++++++++++++++++++++++
 3 files changed, 91 insertions(+), 31 deletions(-)
 create mode 100644 cmd/qualify/assets.go

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
new file mode 100644
index 000000000..fac3c6485
--- /dev/null
+++ b/cmd/qualify/assets.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"os"
+	"strings"
+
+	packr "github.com/gobuffalo/packr/v2"
+	trace "github.com/google/go-trace"
+)
+
+var AcceptanceSuite *packr.Box
+
+func GetAssets() {
+	AcceptanceSuite = packr.New("acceptance suite", "../../acceptance")
+
+	trace.Trace("packr suite: %v", AcceptanceSuite)
+	contents := AcceptanceSuite.List()
+	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
+}
+
+type FilesByDir struct {
+	Directory string
+	Files     []string
+}
+
+func GetFilesByDir(box *packr.Box) []*FilesByDir {
+	filesInAllDirs := []*FilesByDir{}
+	var filesInThisDir *FilesByDir
+	commitThisDir := func() {
+		if filesInThisDir != nil {
+			filesInAllDirs = append(filesInAllDirs, filesInThisDir)
+			filesInThisDir = nil
+		}
+	}
+
+	// We assume allFiles is returned in a top-down order, so that
+	// all files under each first level directry appear
+	// contiguously.
+	allFiles := box.List()
+	previousDir := ""
+	for _, oneFile := range allFiles {
+		dir := strings.Split(oneFile, string(os.PathSeparator))[0]
+		if dir != previousDir {
+			commitThisDir()
+			filesInThisDir = &FilesByDir{Directory: dir}
+		}
+		previousDir = dir
+		filesInThisDir.Files = append(filesInThisDir.Files, oneFile)
+	}
+	commitThisDir()
+
+	return filesInAllDirs
+}
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index fd8635665..80808b167 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -2,11 +2,9 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 
 	trace "github.com/google/go-trace"
@@ -41,9 +39,9 @@ func main() {
 	)
 
 	debugMe := true // TODO: get from CLI args
-	if debugMe {
-		trace.On(true)
-	}
+	trace.On(debugMe)
+
+	GetAssets()
 
 	if err := checkDependencies(); err != nil {
 		os.Exit(RetCodeFailedDependencies)
@@ -84,6 +82,7 @@ func checkDependencies() error {
 	notFound := []string{}
 	trace.Trace("")
 
+	// TODO: Run from this repo, rather than requiring external dependency
 	showcasePath, err := exec.LookPath(showcaseCmd)
 	if err != nil {
 		notFound = append(notFound, showcaseCmd)
@@ -123,37 +122,16 @@ func getGeneratorData() (*GeneratorInfo, error) {
 // GetTestSuites returns a list of Suite as found in the specified
 // suite root directory.
 func GetTestSuites(generator *GeneratorInfo) ([]*Suite, error) {
-	suiteRootPath := "../../acceptance"
-
-	trace.Trace("(generator=%#v)", generator)
 	allSuites := []*Suite{}
-	suiteEntries, err := ioutil.ReadDir(suiteRootPath)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, suiteDir := range suiteEntries {
-		if !suiteDir.IsDir() {
-			continue
-		}
-		name := suiteDir.Name()
-		location, err := filepath.Abs(path.Join(suiteRootPath, name))
-		if err != nil {
-			return nil, err
-		}
-
-		suiteFiles, err := getAllFiles(location)
-		if err != nil {
-			return nil, err
-		}
-
+	allSuiteConfigs := GetFilesByDir(AcceptanceSuite)
+	for _, config := range allSuiteConfigs {
 		defaultShowcasePort := 123 // TODO fix
 		newSuite := &Suite{
-			name:         name,
-			location:     location,
+			name: config.Directory,
+			//			location:     location,
 			showcasePort: defaultShowcasePort,
 			generator:    generator,
-			files:        suiteFiles,
+			files:        config.Files,
 		}
 		trace.Trace("adding suite %#v", newSuite)
 		allSuites = append(allSuites, newSuite)
diff --git a/go.sum b/go.sum
index d5dcab163..5aaadfd82 100644
--- a/go.sum
+++ b/go.sum
@@ -33,6 +33,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -45,6 +46,15 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8=
+github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w=
+github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg=
+github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs=
+github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4=
+github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q=
+github.com/gobuffalo/packr/v2 v2.7.0 h1:snQei95nVOAtgY8rMiccTIPPYUXkuQws34FM/9/+UVg=
+github.com/gobuffalo/packr/v2 v2.7.0/go.mod h1:I+ICQFYFNPqTT7CUw7/RfMhluz683BO8FLH0wZgh/pQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -90,12 +100,17 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.11.3 h1:ZrtYOzzHRzItdU1MvkK3CLlhC4m3YTWFgGyiBuSCQSY=
+github.com/karrick/godirwalk v1.11.3/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -125,9 +140,15 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY=
+github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
@@ -144,9 +165,12 @@ github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -164,6 +188,8 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
+golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -205,6 +231,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -214,8 +241,10 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From c49b73d4eddfd4287c97553084682bcd6d9d4925 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Thu, 3 Oct 2019 16:56:46 -0700
Subject: [PATCH 13/39] Rename "Suite" to "Scenario"

A Suite is a collection of Scenarios
---
 cmd/qualify/qualify.go                | 36 +++++++++++++--------------
 cmd/qualify/{suite.go => scenario.go} | 35 +++++++++++++-------------
 2 files changed, 36 insertions(+), 35 deletions(-)
 rename cmd/qualify/{suite.go => scenario.go} (67%)

diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 80808b167..1ef13e17c 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -35,7 +35,7 @@ func main() {
 		RetCodeFailedDependencies
 		RetCodeUsageError
 		RetCodeCantRunShowcase
-		RetSuiteFailure
+		RetScenarioFailure
 	)
 
 	debugMe := true // TODO: get from CLI args
@@ -58,22 +58,22 @@ func main() {
 		os.Exit(RetCodeUsageError)
 	}
 
-	allSuites, err := GetTestSuites(generatorData)
+	allScenarios, err := GetTestScenarios(generatorData)
 	if err != nil {
 		os.Exit(RetCodeInternalError)
 	}
 
 	success := true
-	for _, suite := range allSuites {
-		if err := suite.Run(); err != nil {
+	for _, scenario := range allScenarios {
+		if err := scenario.Run(); err != nil {
 			os.Exit(RetCodeInternalError)
 		}
-		if !suite.Success() {
+		if !scenario.Success() {
 			success = false
 		}
 	}
 	if !success {
-		os.Exit(RetSuiteFailure)
+		os.Exit(RetScenarioFailure)
 	}
 }
 
@@ -119,24 +119,24 @@ func getGeneratorData() (*GeneratorInfo, error) {
 	return generator, nil
 }
 
-// GetTestSuites returns a list of Suite as found in the specified
-// suite root directory.
-func GetTestSuites(generator *GeneratorInfo) ([]*Suite, error) {
-	allSuites := []*Suite{}
-	allSuiteConfigs := GetFilesByDir(AcceptanceSuite)
-	for _, config := range allSuiteConfigs {
+// GetTestScenarios returns a list of Scenario as found in the specified
+// scenario root directory.
+func GetTestScenarios(generator *GeneratorInfo) ([]*Scenario, error) {
+	allScenarios := []*Scenario{}
+	allScenarioConfigs := GetFilesByDir(AcceptanceSuite)
+	for _, config := range allScenarioConfigs {
 		defaultShowcasePort := 123 // TODO fix
-		newSuite := &Suite{
-			name: config.Directory,
-			//			location:     location,
+		newScenario := &Scenario{
+			name:         config.Directory,
 			showcasePort: defaultShowcasePort,
 			generator:    generator,
 			files:        config.Files,
+			fileBox:      AcceptanceSuite,
 		}
-		trace.Trace("adding suite %#v", newSuite)
-		allSuites = append(allSuites, newSuite)
+		trace.Trace("adding scenario %#v", newScenario)
+		allScenarios = append(allScenarios, newScenario)
 	}
-	return allSuites, nil
+	return allScenarios, nil
 }
 
 // startShowcase starts the Showcase server and returns its PID
diff --git a/cmd/qualify/suite.go b/cmd/qualify/scenario.go
similarity index 67%
rename from cmd/qualify/suite.go
rename to cmd/qualify/scenario.go
index b96d2431e..dfab1b4d1 100644
--- a/cmd/qualify/suite.go
+++ b/cmd/qualify/scenario.go
@@ -6,14 +6,15 @@ import (
 	"io/ioutil"
 	"path/filepath"
 
+	packr "github.com/gobuffalo/packr/v2"
 	trace "github.com/google/go-trace"
 	yaml "gopkg.in/yaml.v2"
 )
 
-type Suite struct {
+type Scenario struct {
 	name        string
-	location    string
 	files       []string
+	fileBox     *packr.Box
 	sandbox     string
 	filesByType map[string][]string
 
@@ -21,43 +22,43 @@ type Suite struct {
 	showcasePort int
 }
 
-func (suite *Suite) Run() error {
-	if err := suite.Generate(); err != nil {
+func (scenario *Scenario) Run() error {
+	if err := scenario.Generate(); err != nil {
 		return err
 	}
-	suite.CheckGeneration()
-	suite.RunTests()
+	scenario.CheckGeneration()
+	scenario.RunTests()
 	return nil
 }
 
-func (suite *Suite) Generate() error {
-	if err := suite.getGenerationFiles(); err != nil {
+func (scenario *Scenario) Generate() error {
+	if err := scenario.getGenerationFiles(); err != nil {
 		return err
 	}
-	suite.sandbox = createSandbox()
+	scenario.sandbox = createSandbox()
 
 	trace.Trace("Generate (TODO)")
 	return nil
 }
 
-func (suite *Suite) CheckGeneration() {
+func (scenario *Scenario) CheckGeneration() {
 	// TODO: Fill in
 	trace.Trace("CheckGeneration (TODO)")
 }
 
-func (suite *Suite) RunTests() {
+func (scenario *Scenario) RunTests() {
 	// TODO: Fill in
 	trace.Trace("RunTests (TODO)")
 }
 
-func (suite *Suite) Success() bool {
+func (scenario *Scenario) Success() bool {
 	// TODO: fill in
 	return true
 }
 
-func (suite *Suite) getGenerationFiles() (err error) {
-	suite.filesByType = make(map[string][]string)
-	for _, thisFile := range suite.files {
+func (scenario *Scenario) getGenerationFiles() (err error) {
+	scenario.filesByType = make(map[string][]string)
+	for _, thisFile := range scenario.files {
 		extension := filepath.Ext(thisFile)
 		var fileTypes []string
 		switch extension {
@@ -78,8 +79,8 @@ func (suite *Suite) getGenerationFiles() (err error) {
 		}
 
 		for _, oneType := range fileTypes {
-			similarFiles := suite.filesByType[oneType]
-			suite.filesByType[oneType] = append(similarFiles, thisFile)
+			similarFiles := scenario.filesByType[oneType]
+			scenario.filesByType[oneType] = append(similarFiles, thisFile)
 			trace.Trace("%s: type %q", thisFile, oneType)
 		}
 	}

From 3baf0acb6c484bd7cec4c0088b2e73cbe414e08d Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Thu, 3 Oct 2019 17:43:33 -0700
Subject: [PATCH 14/39] Add some TODOs

---
 cmd/qualify/scenario.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index dfab1b4d1..0d98d71dd 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -37,6 +37,12 @@ func (scenario *Scenario) Generate() error {
 	}
 	scenario.sandbox = createSandbox()
 
+	/* TODO:
+	   - P1. Invoke sample generator with all the protos in the directory
+	   - P2. when creating sandbox, expand files of the form `include.FOO` to be file/dir FOO/ with contents cloned from the location specified in the file
+	        can be a path rooted at schema/
+	        can be an http: which is curled
+	*/
 	trace.Trace("Generate (TODO)")
 	return nil
 }
@@ -58,6 +64,7 @@ func (scenario *Scenario) Success() bool {
 
 func (scenario *Scenario) getGenerationFiles() (err error) {
 	scenario.filesByType = make(map[string][]string)
+	// TODO: process `include.*` files first
 	for _, thisFile := range scenario.files {
 		extension := filepath.Ext(thisFile)
 		var fileTypes []string
@@ -110,5 +117,6 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 
 func createSandbox() string {
 	// TODO: make a temporary directory that can be deleted
+	// TODO: expand `include.*` files (passed in as params)
 	return "/tmp"
 }

From feec34441f542221bd9d2bff4af96333238e0db8 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 4 Oct 2019 15:22:07 -0700
Subject: [PATCH 15/39] Create sandbox directory with a per-run timestamp in
 its name

Also fix a bug where we were not crrectly retrieving files from packr.Box
---
 cmd/qualify/assets.go   |  2 +-
 cmd/qualify/qualify.go  |  1 +
 cmd/qualify/scenario.go | 96 ++++++++++++++++++++++++++++++++++++-----
 3 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index fac3c6485..e1ccf0944 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -34,7 +34,7 @@ func GetFilesByDir(box *packr.Box) []*FilesByDir {
 	}
 
 	// We assume allFiles is returned in a top-down order, so that
-	// all files under each first level directry appear
+	// all files under each first level directory appear
 	// contiguously.
 	allFiles := box.List()
 	previousDir := ""
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 1ef13e17c..030cc17f1 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -40,6 +40,7 @@ func main() {
 
 	debugMe := true // TODO: get from CLI args
 	trace.On(debugMe)
+	trace.Trace("timestamp: %s", timestamp)
 
 	GetAssets()
 
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 0d98d71dd..979d13082 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -2,16 +2,29 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	packr "github.com/gobuffalo/packr/v2"
 	trace "github.com/google/go-trace"
 	yaml "gopkg.in/yaml.v2"
 )
 
+var timestamp string
+
+func init() {
+	timestamp = time.Now().Format("20060102.150405")
+	trace.Trace("timestamp = %q", timestamp)
+}
+
 type Scenario struct {
+	// Every f in `files` must begin with `name`:
+	// strings.HasPrefix(f, name)
 	name        string
 	files       []string
 	fileBox     *packr.Box
@@ -22,6 +35,31 @@ type Scenario struct {
 	showcasePort int
 }
 
+func (scenario *Scenario) sandboxFileName(srcPath string) string {
+	if len(scenario.name) == 0 {
+		trace.Trace("scenario name not set yet when trying to get sandbox name for %q!", srcPath)
+		panic("scenario name not set yet when trying to get sandbox name")
+	}
+	if len(scenario.sandbox) == 0 {
+		trace.Trace("sandbox not set yet when trying to get sandbox name for %q!", srcPath)
+		panic("sandbox not set yet when trying to get sandbox name")
+	}
+	if !strings.HasPrefix(srcPath, scenario.name) {
+		trace.Trace("file %q is not part of sandbox %q", srcPath, scenario.sandbox)
+		panic("file not part of sandbox")
+	}
+	trace.Trace("srcPath: %s", srcPath)
+
+	relativePath := srcPath[len(scenario.name):]
+	if len(relativePath) > 0 {
+		// There are other path elements, so remove the leading PathSeparator
+		relativePath = relativePath[len(string(os.PathSeparator)):]
+	}
+	dstPath := filepath.Join(scenario.sandbox, relativePath)
+
+	return dstPath
+}
+
 func (scenario *Scenario) Run() error {
 	if err := scenario.Generate(); err != nil {
 		return err
@@ -31,11 +69,15 @@ func (scenario *Scenario) Run() error {
 	return nil
 }
 
-func (scenario *Scenario) Generate() error {
-	if err := scenario.getGenerationFiles(); err != nil {
+func (scenario *Scenario) Generate() (err error) {
+	if err = scenario.getGenerationFiles(); err != nil {
 		return err
 	}
-	scenario.sandbox = createSandbox()
+	err = scenario.createSandbox()
+	if err != nil {
+		return fmt.Errorf("could not create sandbox: %w", err)
+	}
+	trace.Trace("created sandbox: %q", scenario.sandbox)
 
 	/* TODO:
 	   - P1. Invoke sample generator with all the protos in the directory
@@ -74,9 +116,10 @@ func (scenario *Scenario) getGenerationFiles() (err error) {
 			fileTypes = []string{"proto"}
 		case ".yaml", ".yml":
 			trace.Trace("reading %s", thisFile)
-			content, err := ioutil.ReadFile(thisFile)
+			content, err := scenario.fileBox.Find(thisFile)
 			if err != nil {
-				trace.Trace("error reading: %s", err)
+				err := fmt.Errorf("error reading %q: %w", thisFile, err)
+				trace.Trace(err)
 				return err
 			}
 			fileTypes, err = yamlDocTypes(content)
@@ -107,7 +150,8 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 			break
 		}
 		if err != nil {
-			trace.Trace("error decoding: %s", err)
+			err = fmt.Errorf("error decoding file: %w", err)
+			trace.Trace(err)
 			return
 		}
 		docTypes = append(docTypes, doc.Type)
@@ -115,8 +159,40 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 	return docTypes, nil
 }
 
-func createSandbox() string {
-	// TODO: make a temporary directory that can be deleted
-	// TODO: expand `include.*` files (passed in as params)
-	return "/tmp"
+func (scenario *Scenario) createSandbox() (err error) {
+	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", timestamp, scenario.name)); err != nil {
+		return err
+	}
+
+	// Copy files needed to generate
+	for _, srcFile := range scenario.files {
+
+		// TODO: expand `include.*` files (passed in as params)
+
+		srcDir := filepath.Dir(srcFile)
+		dstDir := scenario.sandboxFileName(srcDir)
+		trace.Trace("  srcFile: %s", srcFile)
+		trace.Trace("   srcDir: %s", srcDir)
+		trace.Trace("     name: %s", scenario.name)
+		trace.Trace("   dstDir: %s", dstDir)
+		if _, err := os.Stat(dstDir); os.IsNotExist(err) {
+			if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
+				trace.Trace("could not MkdirAll(%q)", dstDir)
+				return err
+			}
+		}
+
+		var contents []byte
+		if contents, err = scenario.fileBox.Find(srcFile); err != nil {
+			trace.Trace("could not find file %q", srcFile)
+			return err
+		}
+
+		dstFile := scenario.sandboxFileName(srcFile)
+		if err := ioutil.WriteFile(dstFile, contents, 0555); err != nil {
+			trace.Trace("could not WriteFile()")
+			return err
+		}
+	}
+	return nil
 }

From 683c0ff6a94b1b7fb47dc233ce17a092864756c3 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 4 Oct 2019 17:25:43 -0700
Subject: [PATCH 16/39] WIP: Running protoc

Need to store files as relative directories
---
 cmd/qualify/generator.go | 37 +++++++++++++++++++++++++++++++++++++
 cmd/qualify/scenario.go  |  5 +++++
 2 files changed, 42 insertions(+)

diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
index cc3b74fef..e47a64dcf 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/qualify/generator.go
@@ -1,8 +1,45 @@
 package main
 
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	trace "github.com/google/go-trace"
+)
+
 type GeneratorInfo struct {
 	name       string
 	dir        string
 	options    string
 	isMonolith bool
 }
+
+func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]byte, error) {
+	if gi.isMonolith {
+		return nil, fmt.Errorf("monolith not implemented yet")
+	}
+
+	//	allProtos := strings.Join(filesByType["proto"], " ")
+	cmdParts := []string{
+		"protoc",
+		fmt.Sprintf("--%s_gapic_out", gi.name),
+		"./gen",
+		fmt.Sprintf("--%s_gapic_opt", gi.name),
+		gi.options,
+		fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name),
+	}
+	cmdParts = append(cmdParts, filesByType["proto"]...)
+	cmdString := strings.Join(cmdParts, " ")
+
+	// MAYBE we need to store the list of files as relative dirs, and keep the source dir separate
+
+	// cmdString := fmt.Sprintf("protoc --%s_gapic_out ./gen --%s_gapic_opt %s --plugin=%s/protoc-gen-%s_gapic %s",
+	// 	gi.name, gi.name, gi.options,
+	// 	gi.dir, gi.name,
+	// 	allProtos)
+	trace.Trace("running: %s", cmdString)
+	cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
+	cmd.Dir = wdir
+	return cmd.CombinedOutput()
+}
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 979d13082..fb22b7e7c 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -79,6 +79,11 @@ func (scenario *Scenario) Generate() (err error) {
 	}
 	trace.Trace("created sandbox: %q", scenario.sandbox)
 
+	var output []byte
+	output, err = scenario.generator.Run(scenario.sandbox, scenario.filesByType)
+	trace.Trace("run error: %s", err)
+	trace.Trace("run output: %s", output)
+
 	/* TODO:
 	   - P1. Invoke sample generator with all the protos in the directory
 	   - P2. when creating sandbox, expand files of the form `include.FOO` to be file/dir FOO/ with contents cloned from the location specified in the file

From 73b3768b9245dfb1371aef38a39c31e738ec3bb6 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 5 Oct 2019 08:22:20 -0700
Subject: [PATCH 17/39] Make the file list stored with each scenario be
 relative paths

The paths are relative to the scenario dir (in either the source file
box or in the sandbox)
---
 cmd/qualify/assets.go   |  4 +--
 cmd/qualify/scenario.go | 59 +++++++++++++++++++++--------------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index e1ccf0944..25371a050 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -20,7 +20,7 @@ func GetAssets() {
 
 type FilesByDir struct {
 	Directory string
-	Files     []string
+	Files     []string // filenames are relative paths from `Directory`
 }
 
 func GetFilesByDir(box *packr.Box) []*FilesByDir {
@@ -45,7 +45,7 @@ func GetFilesByDir(box *packr.Box) []*FilesByDir {
 			filesInThisDir = &FilesByDir{Directory: dir}
 		}
 		previousDir = dir
-		filesInThisDir.Files = append(filesInThisDir.Files, oneFile)
+		filesInThisDir.Files = append(filesInThisDir.Files, oneFile[len(dir)+len(string(os.PathSeparator)):])
 	}
 	commitThisDir()
 
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index fb22b7e7c..0ab86a949 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	packr "github.com/gobuffalo/packr/v2"
@@ -35,29 +34,29 @@ type Scenario struct {
 	showcasePort int
 }
 
-func (scenario *Scenario) sandboxFileName(srcPath string) string {
+func (scenario *Scenario) sandboxPath(relativePath string) string {
 	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet when trying to get sandbox name for %q!", srcPath)
-		panic("scenario name not set yet when trying to get sandbox name")
+		trace.Trace("scenario name not set yet: relativePath==%q!", relativePath)
+		panic("scenario name not set yet when trying to get sandbox filename")
 	}
 	if len(scenario.sandbox) == 0 {
-		trace.Trace("sandbox not set yet when trying to get sandbox name for %q!", srcPath)
-		panic("sandbox not set yet when trying to get sandbox name")
+		trace.Trace("sandbox name not set yet: relativePath==%q!", relativePath)
+		panic("sandbox name not set yet when trying to get sandbox filename")
 	}
-	if !strings.HasPrefix(srcPath, scenario.name) {
-		trace.Trace("file %q is not part of sandbox %q", srcPath, scenario.sandbox)
-		panic("file not part of sandbox")
-	}
-	trace.Trace("srcPath: %s", srcPath)
 
-	relativePath := srcPath[len(scenario.name):]
-	if len(relativePath) > 0 {
-		// There are other path elements, so remove the leading PathSeparator
-		relativePath = relativePath[len(string(os.PathSeparator)):]
+	return filepath.Join(scenario.sandbox, relativePath)
+}
+
+func (scenario *Scenario) fileBoxPath(relativePath string) string {
+	if len(scenario.name) == 0 {
+		trace.Trace("scenario name not set yet: relativePath==%q!", relativePath)
+		panic("scenario name not set yet when trying to get box filename")
 	}
-	dstPath := filepath.Join(scenario.sandbox, relativePath)
+	return filepath.Join(scenario.name, relativePath)
+}
 
-	return dstPath
+func (scenario *Scenario) getBoxFile(relativePath string) ([]byte, error) {
+	return scenario.fileBox.Find(scenario.fileBoxPath(relativePath))
 }
 
 func (scenario *Scenario) Run() error {
@@ -121,7 +120,7 @@ func (scenario *Scenario) getGenerationFiles() (err error) {
 			fileTypes = []string{"proto"}
 		case ".yaml", ".yml":
 			trace.Trace("reading %s", thisFile)
-			content, err := scenario.fileBox.Find(thisFile)
+			content, err := scenario.getBoxFile(thisFile)
 			if err != nil {
 				err := fmt.Errorf("error reading %q: %w", thisFile, err)
 				trace.Trace(err)
@@ -165,6 +164,8 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 }
 
 func (scenario *Scenario) createSandbox() (err error) {
+	const filePermissions = 0555
+
 	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", timestamp, scenario.name)); err != nil {
 		return err
 	}
@@ -175,29 +176,29 @@ func (scenario *Scenario) createSandbox() (err error) {
 		// TODO: expand `include.*` files (passed in as params)
 
 		srcDir := filepath.Dir(srcFile)
-		dstDir := scenario.sandboxFileName(srcDir)
-		trace.Trace("  srcFile: %s", srcFile)
-		trace.Trace("   srcDir: %s", srcDir)
-		trace.Trace("     name: %s", scenario.name)
-		trace.Trace("   dstDir: %s", dstDir)
+		dstDir := scenario.sandboxPath(srcDir)
 		if _, err := os.Stat(dstDir); os.IsNotExist(err) {
 			if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
-				trace.Trace("could not MkdirAll(%q)", dstDir)
+				err = fmt.Errorf("could not make directory %q: %w", dstDir, err)
+				trace.Trace(err)
 				return err
 			}
 		}
 
 		var contents []byte
-		if contents, err = scenario.fileBox.Find(srcFile); err != nil {
-			trace.Trace("could not find file %q", srcFile)
+		if contents, err = scenario.getBoxFile(srcFile); err != nil {
+			err = fmt.Errorf("could not find file %q: %w", srcFile, err)
+			trace.Trace(err)
 			return err
 		}
 
-		dstFile := scenario.sandboxFileName(srcFile)
-		if err := ioutil.WriteFile(dstFile, contents, 0555); err != nil {
-			trace.Trace("could not WriteFile()")
+		dstFile := scenario.sandboxPath(srcFile)
+		if err := ioutil.WriteFile(dstFile, contents, filePermissions); err != nil {
+			err = fmt.Errorf("could not write file  %q: %w", dstFile, err)
+			trace.Trace(err)
 			return err
 		}
+		trace.Trace(dstFile)
 	}
 	return nil
 }

From 60456c2d555c7e1bae9ed34dae94ef9a78a1d983 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 5 Oct 2019 08:32:45 -0700
Subject: [PATCH 18/39] Cosmetic changes

protoc still not working
---
 cmd/qualify/generator.go | 6 ++----
 cmd/qualify/qualify.go   | 6 +++---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
index e47a64dcf..a1b4ad619 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/qualify/generator.go
@@ -23,10 +23,8 @@ func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]by
 	//	allProtos := strings.Join(filesByType["proto"], " ")
 	cmdParts := []string{
 		"protoc",
-		fmt.Sprintf("--%s_gapic_out", gi.name),
-		"./gen",
-		fmt.Sprintf("--%s_gapic_opt", gi.name),
-		gi.options,
+		fmt.Sprintf("--%s_gapic_out", gi.name), "./generated",
+		fmt.Sprintf("--%s_gapic_opt", gi.name), gi.options,
 		fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name),
 	}
 	cmdParts = append(cmdParts, filesByType["proto"]...)
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 030cc17f1..2fcce6afe 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -108,9 +108,9 @@ func checkDependencies() error {
 // line, and whether it is a protoc plugin (if not, it is considered
 // part of the monolith)
 func getGeneratorData() (*GeneratorInfo, error) {
-	pluginName := "go"                                                                 // TODO: get from CLI args
-	pluginDir := "/tmp/goinstall/bin"                                                  // TODO: get from CLI args
-	pluginOpt := "'go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase'" // TODO: get from CLI args
+	pluginName := "go"                                                               // TODO: get from CLI args
+	pluginDir := "/tmp/goinstall/bin"                                                // TODO: get from CLI args
+	pluginOpt := "go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase" // TODO: get from CLI args
 	generator := &GeneratorInfo{
 		name:    pluginName,
 		dir:     pluginDir,

From ff2b11742ef0487629fd4322f2d4618a181bbc80 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 7 Oct 2019 16:01:29 -0700
Subject: [PATCH 19/39] Get common proto files in a box

---
 cmd/qualify/assets.go   |  7 ++++++-
 cmd/qualify/scenario.go | 13 ++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index 25371a050..2778751bf 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -9,13 +9,18 @@ import (
 )
 
 var AcceptanceSuite *packr.Box
+var SchemaSuite *packr.Box
 
 func GetAssets() {
 	AcceptanceSuite = packr.New("acceptance suite", "../../acceptance")
-
 	trace.Trace("packr suite: %v", AcceptanceSuite)
 	contents := AcceptanceSuite.List()
 	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
+
+	SchemaSuite = packr.New("schema suite", "../../schema")
+	trace.Trace("packr suite: %v", SchemaSuite)
+	contents = SchemaSuite.List()
+	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
 }
 
 type FilesByDir struct {
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 0ab86a949..2c1933028 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -164,14 +164,21 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 }
 
 func (scenario *Scenario) createSandbox() (err error) {
-	const filePermissions = 0555
 
 	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", timestamp, scenario.name)); err != nil {
 		return err
 	}
 
-	// Copy files needed to generate
-	for _, srcFile := range scenario.files {
+	if err = scenario.copyFiles(scenario.files); err != nil {
+		err := fmt.Errorf("could not copy scenario files: %w", err)
+		return err
+	}
+	return nil
+}
+
+func (scenario *Scenario) copyFiles(files []string) (err error) {
+	const filePermissions = 0555
+	for _, srcFile := range files {
 
 		// TODO: expand `include.*` files (passed in as params)
 

From 508e6c26342ab7476e00b86ce85c7ac35ae936dd Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Tue, 8 Oct 2019 12:44:23 -0700
Subject: [PATCH 20/39] Process top-level .include files and directories

---
 cmd/qualify/assets.go    | 54 ++++++++++++++++++++++-
 cmd/qualify/generator.go | 16 +++----
 cmd/qualify/qualify.go   |  1 +
 cmd/qualify/scenario.go  | 95 +++++++++++++++++++++++++++++++++-------
 4 files changed, 140 insertions(+), 26 deletions(-)

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index 2778751bf..1ed9380d8 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -17,7 +18,7 @@ func GetAssets() {
 	contents := AcceptanceSuite.List()
 	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
 
-	SchemaSuite = packr.New("schema suite", "../../schema")
+	SchemaSuite = packr.New("schema", "../../schema")
 	trace.Trace("packr suite: %v", SchemaSuite)
 	contents = SchemaSuite.List()
 	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
@@ -56,3 +57,54 @@ func GetFilesByDir(box *packr.Box) []*FilesByDir {
 
 	return filesInAllDirs
 }
+
+// GetMatchingFiles returns the a list of files in `box` matching
+// `pattern`. `pattern` should end with `pathSeparator` iff it refers
+// to a directory.
+//
+// This also returns a couple of additional values. These are useful
+// for copying files within matched directories, and are provided as a
+// convenience:
+//
+//  * `srcPath`: a version of `pattern` modified so that
+//    `pathSeparator`, if any, becomes `os.PathSeparator`
+//  * `replacePath`: a version of `dstPath` (which is assumed to have
+//     no trailing separator) with `os.PathSeparator` appended.
+func GetMatchingFiles(box *packr.Box, dstPath string, pattern string, pathSeparator rune) (files []string, srcPath string, replacePath string, err error) {
+
+	trace.Trace("reading %q with separator %c", pattern, pathSeparator)
+
+	replacePath = dstPath
+	srcPath = pattern
+
+	// If `pattern` specifies a single file, match just that.
+	if !strings.HasSuffix(pattern, string(pathSeparator)) {
+		if !box.Has(pattern) {
+			err = fmt.Errorf("file box %q has no file %q", box.Name, pattern)
+			return
+		}
+		files = []string{pattern}
+		return
+	}
+
+	// Replace the trailing pathSeparator (hard-coded in text)
+	// with the os.PathSeparator presumably used by packr, which
+	// varies by OS
+	srcPath = string(append([]rune(srcPath)[:len(srcPath)-1], os.PathSeparator))
+
+	// Likewise, let the caller replace the directory part of the
+	// path with the separator included.
+	replacePath = dstPath + string(os.PathSeparator)
+
+	for _, entry := range box.List() {
+		if strings.HasPrefix(entry, srcPath) {
+			files = append(files, entry)
+		}
+	}
+	if len(files) == 0 {
+		err = fmt.Errorf("file box %q has no files matching %q", box.Name, pattern)
+		return
+	}
+	trace.Trace("obtained files %q", files)
+	return
+}
diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
index a1b4ad619..75369c4d9 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/qualify/generator.go
@@ -2,7 +2,9 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
+	"path"
 	"strings"
 
 	trace "github.com/google/go-trace"
@@ -16,26 +18,24 @@ type GeneratorInfo struct {
 }
 
 func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]byte, error) {
+	generationDir := "generated"
 	if gi.isMonolith {
 		return nil, fmt.Errorf("monolith not implemented yet")
 	}
 
-	//	allProtos := strings.Join(filesByType["proto"], " ")
+	if err := os.MkdirAll(path.Join(wdir, generationDir), os.ModePerm); err != nil {
+		return nil, err
+	}
+
 	cmdParts := []string{
 		"protoc",
-		fmt.Sprintf("--%s_gapic_out", gi.name), "./generated",
+		fmt.Sprintf("--%s_gapic_out", gi.name), fmt.Sprintf("./%s", generationDir),
 		fmt.Sprintf("--%s_gapic_opt", gi.name), gi.options,
 		fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name),
 	}
 	cmdParts = append(cmdParts, filesByType["proto"]...)
 	cmdString := strings.Join(cmdParts, " ")
 
-	// MAYBE we need to store the list of files as relative dirs, and keep the source dir separate
-
-	// cmdString := fmt.Sprintf("protoc --%s_gapic_out ./gen --%s_gapic_opt %s --plugin=%s/protoc-gen-%s_gapic %s",
-	// 	gi.name, gi.name, gi.options,
-	// 	gi.dir, gi.name,
-	// 	allProtos)
 	trace.Trace("running: %s", cmdString)
 	cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
 	cmd.Dir = wdir
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 2fcce6afe..58c9f2d90 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -133,6 +133,7 @@ func GetTestScenarios(generator *GeneratorInfo) ([]*Scenario, error) {
 			generator:    generator,
 			files:        config.Files,
 			fileBox:      AcceptanceSuite,
+			schemaBox:    SchemaSuite,
 		}
 		trace.Trace("adding scenario %#v", newScenario)
 		allScenarios = append(allScenarios, newScenario)
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 2c1933028..0e77e08e5 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	packr "github.com/gobuffalo/packr/v2"
@@ -27,6 +28,7 @@ type Scenario struct {
 	name        string
 	files       []string
 	fileBox     *packr.Box
+	schemaBox   *packr.Box
 	sandbox     string
 	filesByType map[string][]string
 
@@ -55,7 +57,7 @@ func (scenario *Scenario) fileBoxPath(relativePath string) string {
 	return filepath.Join(scenario.name, relativePath)
 }
 
-func (scenario *Scenario) getBoxFile(relativePath string) ([]byte, error) {
+func (scenario *Scenario) fromFileBox(relativePath string) ([]byte, error) {
 	return scenario.fileBox.Find(scenario.fileBoxPath(relativePath))
 }
 
@@ -80,7 +82,7 @@ func (scenario *Scenario) Generate() (err error) {
 
 	var output []byte
 	output, err = scenario.generator.Run(scenario.sandbox, scenario.filesByType)
-	trace.Trace("run error: %s", err)
+	trace.Trace("run error: %v", err)
 	trace.Trace("run output: %s", output)
 
 	/* TODO:
@@ -120,7 +122,7 @@ func (scenario *Scenario) getGenerationFiles() (err error) {
 			fileTypes = []string{"proto"}
 		case ".yaml", ".yml":
 			trace.Trace("reading %s", thisFile)
-			content, err := scenario.getBoxFile(thisFile)
+			content, err := scenario.fromFileBox(thisFile)
 			if err != nil {
 				err := fmt.Errorf("error reading %q: %w", thisFile, err)
 				trace.Trace(err)
@@ -166,46 +168,105 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 func (scenario *Scenario) createSandbox() (err error) {
 
 	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", timestamp, scenario.name)); err != nil {
+		err := fmt.Errorf("could not create sandbox: %w", err)
+		trace.Trace(err)
 		return err
 	}
 
-	if err = scenario.copyFiles(scenario.files); err != nil {
-		err := fmt.Errorf("could not copy scenario files: %w", err)
+	includePaths, err := scenario.copyFiles(scenario.files, true)
+	if err != nil {
+		err = fmt.Errorf("could not copy scenario files: %w", err)
+		trace.Trace(err)
 		return err
 	}
+
+	if err := scenario.getIncludes(includePaths); err != nil {
+		err = fmt.Errorf("could not copy included schema files: %w", err)
+		trace.Trace(err)
+		return err
+	}
+
+	trace.Trace("created sandbox")
 	return nil
 }
 
-func (scenario *Scenario) copyFiles(files []string) (err error) {
+const includeFilePrefix = "include."
+
+func (scenario *Scenario) getIncludes(includes []string) error {
+	for _, includeFile := range includes {
+		dstPath := includeFile[len(includeFilePrefix):]
+		content, err := scenario.fromFileBox(includeFile)
+		if err != nil {
+			err = fmt.Errorf("could not read %q: %w", includeFile, err)
+			trace.Trace(err)
+			return err
+		}
+
+		files, srcPath, replacePath, err := GetMatchingFiles(scenario.schemaBox, dstPath, string(content), '/')
+		if err != nil {
+			err = fmt.Errorf("could not process %q: %w", includeFile, err)
+			trace.Trace(err)
+			return err
+		}
+
+		scenario.copyFilesReplacePath(files, false, scenario.schemaBox.Find, srcPath, replacePath)
+	}
+	return nil
+}
+
+func (scenario *Scenario) copyFiles(files []string, returnIncludes bool) (includes []string, err error) {
+	return scenario.copyFilesReplacePath(files, returnIncludes, scenario.fromFileBox, "", "")
+}
+
+func (scenario *Scenario) copyFilesReplacePath(files []string, returnIncludes bool, fromBox func(string) ([]byte, error), replaceSrc, replaceDst string) (includes []string, err error) {
 	const filePermissions = 0555
+	replace := len(replaceSrc) > 0
+
+	trace.Trace("returnIncludes: %v  replaceSrc:%q  replaceDst:%q  files%q",
+		returnIncludes, replaceSrc, replaceDst, files)
+
 	for _, srcFile := range files {
 
-		// TODO: expand `include.*` files (passed in as params)
+		if returnIncludes && strings.HasPrefix(srcFile, includeFilePrefix) && strings.IndexAny(srcFile, string(os.PathSeparator)) == -1 {
+			// This is an include.* file
+			includes = append(includes, srcFile)
+			continue
+		}
+
+		renamedFile := srcFile
+		if replace {
+			if !strings.HasPrefix(renamedFile, replaceSrc) {
+				panic(fmt.Sprintf("%q does not begin with %q", srcFile, replaceSrc))
+			}
+			renamedFile = strings.Replace(renamedFile, replaceSrc, replaceDst, 1)
+			trace.Trace("renamedFile: %q", renamedFile)
+		}
+
+		renamedDir := filepath.Dir(renamedFile)
 
-		srcDir := filepath.Dir(srcFile)
-		dstDir := scenario.sandboxPath(srcDir)
-		if _, err := os.Stat(dstDir); os.IsNotExist(err) {
+		dstDir := scenario.sandboxPath(renamedDir)
+		if _, err = os.Stat(dstDir); os.IsNotExist(err) {
 			if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
 				err = fmt.Errorf("could not make directory %q: %w", dstDir, err)
 				trace.Trace(err)
-				return err
+				return includes, err
 			}
 		}
 
 		var contents []byte
-		if contents, err = scenario.getBoxFile(srcFile); err != nil {
+		if contents, err = fromBox(srcFile); err != nil {
 			err = fmt.Errorf("could not find file %q: %w", srcFile, err)
 			trace.Trace(err)
-			return err
+			return includes, err
 		}
 
-		dstFile := scenario.sandboxPath(srcFile)
-		if err := ioutil.WriteFile(dstFile, contents, filePermissions); err != nil {
+		dstFile := scenario.sandboxPath(renamedFile)
+		if err = ioutil.WriteFile(dstFile, contents, filePermissions); err != nil {
 			err = fmt.Errorf("could not write file  %q: %w", dstFile, err)
 			trace.Trace(err)
-			return err
+			return includes, err
 		}
 		trace.Trace(dstFile)
 	}
-	return nil
+	return includes, nil
 }

From 85188ef187447b28381c445c324893ceb685a204 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Tue, 8 Oct 2019 15:09:13 -0700
Subject: [PATCH 21/39] Allow include files anywhere in acceptance/ hierarchy
 into sandbox

---
 acceptance/passing/echo.proto             | 211 +++++++++++++++++++++-
 acceptance/passing/foo/bar/include.wdREAD |   1 +
 acceptance/passing/google                 |   1 -
 acceptance/passing/include.google         |   1 +
 acceptance/passing/include.vcREAD         |   1 +
 cmd/qualify/assets.go                     |  43 ++---
 cmd/qualify/scenario.go                   | 130 ++++++++-----
 7 files changed, 314 insertions(+), 74 deletions(-)
 mode change 120000 => 100644 acceptance/passing/echo.proto
 create mode 100644 acceptance/passing/foo/bar/include.wdREAD
 delete mode 120000 acceptance/passing/google
 create mode 100644 acceptance/passing/include.google
 create mode 100644 acceptance/passing/include.vcREAD

diff --git a/acceptance/passing/echo.proto b/acceptance/passing/echo.proto
deleted file mode 120000
index a63c95653..000000000
--- a/acceptance/passing/echo.proto
+++ /dev/null
@@ -1 +0,0 @@
-../../schema/google/showcase/v1beta1/echo.proto
\ No newline at end of file
diff --git a/acceptance/passing/echo.proto b/acceptance/passing/echo.proto
new file mode 100644
index 000000000..06b82b432
--- /dev/null
+++ b/acceptance/passing/echo.proto
@@ -0,0 +1,210 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+import "google/api/annotations.proto";
+import "google/api/client.proto";
+import "google/api/field_behavior.proto";
+import "google/longrunning/operations.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+import "google/rpc/status.proto";
+
+package google.showcase.v1beta1;
+
+option go_package = "github.com/googleapis/gapic-showcase/server/genproto";
+option java_package = "com.google.showcase.v1beta1";
+option java_multiple_files = true;
+
+// This service is used showcase the four main types of rpcs - unary, server
+// side streaming, client side streaming, and bidirectional streaming. This
+// service also exposes methods that explicitly implement server delay, and
+// paginated calls. Set the 'showcase-trailer' metadata key on any method
+// to have the values echoed in the response trailers.
+service Echo {
+  // This service is meant to only run locally on the port 7469 (keypad digits
+  // for "show").
+  option (google.api.default_host) = "localhost:7469";
+
+  // This method simply echos the request. This method is showcases unary rpcs.
+  rpc Echo(EchoRequest) returns (EchoResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:echo"
+      body: "*"
+    };
+  }
+
+  // This method split the given content into words and will pass each word back
+  // through the stream. This method showcases server-side streaming rpcs.
+  rpc Expand(ExpandRequest) returns (stream EchoResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:expand"
+      body: "*"
+    };
+    // TODO(landrito): change this to be `fields: ["content", "error"]` once
+    // github.com/dcodeIO/protobuf.js/issues/1094 has been resolved.
+    option (google.api.method_signature) = "content,error";
+  }
+
+  // This method will collect the words given to it. When the stream is closed
+  // by the client, this method will return the a concatenation of the strings
+  // passed to it. This method showcases client-side streaming rpcs.
+  rpc Collect(stream EchoRequest) returns (EchoResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:collect"
+      body: "*"
+    };
+  }
+
+  // This method, upon receiving a request on the stream, will pass
+  // the same content back on the stream. This method showcases
+  // bidirectional streaming rpcs.
+  rpc Chat(stream EchoRequest) returns (stream EchoResponse);
+
+  // This is similar to the Expand method but instead of returning a stream of
+  // expanded words, this method returns a paged list of expanded words.
+  rpc PagedExpand(PagedExpandRequest) returns (PagedExpandResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:pagedExpand"
+      body: "*"
+    };
+  }
+
+  // This method will wait the requested amount of and then return.
+  // This method showcases how a client handles a request timing out.
+  rpc Wait(WaitRequest) returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:wait"
+      body: "*"
+    };
+    option (google.longrunning.operation_info) = {
+      response_type: "WaitResponse"
+      metadata_type: "WaitMetadata"
+    };
+  }
+
+  // This method will block (wait) for the requested amount of time 
+  // and then return the response or error.
+  // This method showcases how a client handles delays or retries.
+  rpc Block(BlockRequest) returns (BlockResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/echo:block"
+      body: "*"
+    };
+  };
+}
+
+// The request message used for the Echo, Collect and Chat methods. If content
+// is set in this message then the request will succeed. If status is set in 
+// this message then the status will be returned as an error.
+message EchoRequest {
+  oneof response {
+    // The content to be echoed by the server.
+    string content = 1;
+
+    // The error to be thrown by the server.
+    google.rpc.Status error = 2;
+  }
+}
+
+// The response message for the Echo methods.
+message EchoResponse {
+  // The content specified in the request.
+  string content = 1;
+}
+
+// The request message for the Expand method.
+message ExpandRequest {
+  // The content that will be split into words and returned on the stream.
+  string content = 1;
+
+  // The error that is thrown after all words are sent on the stream.
+  google.rpc.Status error = 2;
+}
+
+// The request for the PagedExpand method.
+message PagedExpandRequest {
+  // The string to expand.
+  string content = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // The amount of words to returned in each page.
+  int32 page_size = 2;
+
+  // The position of the page to be returned.
+  string page_token = 3;
+}
+
+// The response for the PagedExpand method.
+message PagedExpandResponse {
+  // The words that were expanded.
+  repeated EchoResponse responses = 1;
+
+  // The next page token.
+  string next_page_token = 2;
+}
+
+// The request for Wait method.
+message WaitRequest {
+  oneof end {
+    // The time that this operation will complete.
+    google.protobuf.Timestamp end_time = 1;
+
+    // The duration of this operation.
+    google.protobuf.Duration ttl = 4;
+  }
+
+  oneof response {
+    // The error that will be returned by the server. If this code is specified
+    // to be the OK rpc code, an empty response will be returned.
+    google.rpc.Status error = 2;
+
+    // The response to be returned on operation completion.
+    WaitResponse success = 3;
+  }
+}
+
+// The result of the Wait operation.
+message WaitResponse {
+  // This content of the result.
+  string content = 1;
+}
+
+// The metadata for Wait operation.
+message WaitMetadata {
+  // The time that this operation will complete.
+  google.protobuf.Timestamp end_time =1;
+}
+
+// The request for Block method.
+message BlockRequest {
+  // The amount of time to block before returning a response.
+  google.protobuf.Duration response_delay = 1;
+
+  oneof response {
+    // The error that will be returned by the server. If this code is specified
+    // to be the OK rpc code, an empty response will be returned.
+    google.rpc.Status error = 2;
+
+    // The response to be returned that will signify successful method call.
+    BlockResponse success = 3;
+  }
+}
+
+// The response for Block method.
+message BlockResponse {
+  // This content can contain anything, the server will not depend on a value
+  // here.
+  string content = 1;
+}
diff --git a/acceptance/passing/foo/bar/include.wdREAD b/acceptance/passing/foo/bar/include.wdREAD
new file mode 100644
index 000000000..be3faf98a
--- /dev/null
+++ b/acceptance/passing/foo/bar/include.wdREAD
@@ -0,0 +1 @@
+api-common-protos/CONTRIBUTING.md
diff --git a/acceptance/passing/google b/acceptance/passing/google
deleted file mode 120000
index 82d2520f7..000000000
--- a/acceptance/passing/google
+++ /dev/null
@@ -1 +0,0 @@
-../../schema/api-common-protos/google
\ No newline at end of file
diff --git a/acceptance/passing/include.google b/acceptance/passing/include.google
new file mode 100644
index 000000000..e2f99f6b0
--- /dev/null
+++ b/acceptance/passing/include.google
@@ -0,0 +1 @@
+api-common-protos/google/
\ No newline at end of file
diff --git a/acceptance/passing/include.vcREAD b/acceptance/passing/include.vcREAD
new file mode 100644
index 000000000..332a237b8
--- /dev/null
+++ b/acceptance/passing/include.vcREAD
@@ -0,0 +1 @@
+api-common-protos/README.md
\ No newline at end of file
diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index 1ed9380d8..4aec55a1c 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -59,41 +59,33 @@ func GetFilesByDir(box *packr.Box) []*FilesByDir {
 }
 
 // GetMatchingFiles returns the a list of files in `box` matching
-// `pattern`. `pattern` should end with `pathSeparator` iff it refers
+// `srcPath`. `srcPath` should end with `os.PathSeparator` iff it refers
 // to a directory.
 //
-// This also returns a couple of additional values. These are useful
-// for copying files within matched directories, and are provided as a
-// convenience:
+// As a convenience, this also returns an additional value, useful
+// for copying files within matched directories:
 //
-//  * `srcPath`: a version of `pattern` modified so that
-//    `pathSeparator`, if any, becomes `os.PathSeparator`
-//  * `replacePath`: a version of `dstPath` (which is assumed to have
-//     no trailing separator) with `os.PathSeparator` appended.
-func GetMatchingFiles(box *packr.Box, dstPath string, pattern string, pathSeparator rune) (files []string, srcPath string, replacePath string, err error) {
+//  * `replacePath`: a copy of `dstPath` (which is assumed to have no
+//     trailing separator), with `os.PathSeparator` appended if
+//     `srcPath` refers to a directory
+func GetMatchingFiles(box *packr.Box, dstPath string, srcPath string) (files []string, replacePath string, err error) {
 
-	trace.Trace("reading %q with separator %c", pattern, pathSeparator)
+	trace.Trace("reading %q", srcPath)
 
 	replacePath = dstPath
-	srcPath = pattern
 
-	// If `pattern` specifies a single file, match just that.
-	if !strings.HasSuffix(pattern, string(pathSeparator)) {
-		if !box.Has(pattern) {
-			err = fmt.Errorf("file box %q has no file %q", box.Name, pattern)
+	// If `srcPath` specifies a single file, match just that.
+	if !strings.HasSuffix(srcPath, string(os.PathSeparator)) {
+		if !box.Has(srcPath) {
+			err = fmt.Errorf("file box %q has no file %q", box.Name, srcPath)
 			return
 		}
-		files = []string{pattern}
+		files = []string{srcPath}
 		return
 	}
 
-	// Replace the trailing pathSeparator (hard-coded in text)
-	// with the os.PathSeparator presumably used by packr, which
-	// varies by OS
-	srcPath = string(append([]rune(srcPath)[:len(srcPath)-1], os.PathSeparator))
-
-	// Likewise, let the caller replace the directory part of the
-	// path with the separator included.
+	// Let the caller replace the directory part of the path with
+	// the separator included.
 	replacePath = dstPath + string(os.PathSeparator)
 
 	for _, entry := range box.List() {
@@ -102,9 +94,10 @@ func GetMatchingFiles(box *packr.Box, dstPath string, pattern string, pathSepara
 		}
 	}
 	if len(files) == 0 {
-		err = fmt.Errorf("file box %q has no files matching %q", box.Name, pattern)
+		err = fmt.Errorf("file box %q has no files matching %q", box.Name, srcPath)
 		return
 	}
-	trace.Trace("obtained files %q", files)
+
+	trace.Trace("obtained %d files", len(files))
 	return
 }
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 0e77e08e5..6ad5ab8c5 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -17,14 +17,18 @@ import (
 
 var timestamp string
 
+const includeFilePrefix = "include."
+const includeFileDirectorySeparator = '/'
+
 func init() {
 	timestamp = time.Now().Format("20060102.150405")
 	trace.Trace("timestamp = %q", timestamp)
 }
 
 type Scenario struct {
-	// Every f in `files` must begin with `name`:
-	// strings.HasPrefix(f, name)
+	// Every f in `files` corresponds to a file in `fileBox` under
+	// a directory called `name`, which is NOT part of the string
+	// f itself.
 	name        string
 	files       []string
 	fileBox     *packr.Box
@@ -110,39 +114,71 @@ func (scenario *Scenario) Success() bool {
 	return true
 }
 
+// getGenerationFiles classifies all the files in the scenario,
+// storing the results in `scenario.filesByType`. Each file is
+// classified as either an "include" file or a "scenario"
+// file. Scenario files are also classified by the type of data they
+// contain. Thus, "scenario" files have at least two labels (ie
+// multiple entries in `scenario.filesByType`).
 func (scenario *Scenario) getGenerationFiles() (err error) {
 	scenario.filesByType = make(map[string][]string)
 	// TODO: process `include.*` files first
 	for _, thisFile := range scenario.files {
-		extension := filepath.Ext(thisFile)
-		var fileTypes []string
-		switch extension {
-
-		case ".proto":
-			fileTypes = []string{"proto"}
-		case ".yaml", ".yml":
-			trace.Trace("reading %s", thisFile)
-			content, err := scenario.fromFileBox(thisFile)
-			if err != nil {
-				err := fmt.Errorf("error reading %q: %w", thisFile, err)
-				trace.Trace(err)
-				return err
-			}
-			fileTypes, err = yamlDocTypes(content)
-			if err != nil {
-				return err
-			}
+		fileTypes, err := scenario.getFileTypes(thisFile)
+		if err != nil {
+			return err
 		}
-
 		for _, oneType := range fileTypes {
 			similarFiles := scenario.filesByType[oneType]
 			scenario.filesByType[oneType] = append(similarFiles, thisFile)
-			trace.Trace("%s: type %q", thisFile, oneType)
 		}
+		trace.Trace("%s: type %q", thisFile, fileTypes)
 	}
 	return nil
 }
 
+// getFileTypes gets the various type labels for `thisFile`. The type
+// labels are either the singleton list {"include"}, or a list
+// {"scenario", ...} that includes the various types of data included
+// in that file.
+func (scenario *Scenario) getFileTypes(thisFile string) (fileTypes []string, err error) {
+
+	for idx := strings.Index(thisFile, includeFilePrefix); idx >= 0; idx = strings.Index(thisFile[idx+1:], includeFilePrefix) {
+		if !(idx == 0 || thisFile[idx-1] == os.PathSeparator) {
+			// The directory entry does not start with
+			// this pattern, so it's not an include file.
+			continue
+		}
+		if strings.Index(thisFile[idx+1:], string(os.PathListSeparator)) != -1 {
+			// There is a sub-directory component, so this is not a file.
+			continue
+		}
+
+		// This looks like a legitimate include file. Cease classifying.
+		return []string{"include"}, nil
+	}
+
+	extension := filepath.Ext(thisFile)
+	switch extension {
+	case ".proto":
+		fileTypes = []string{"proto"}
+	case ".yaml", ".yml":
+		trace.Trace("reading %s", thisFile)
+		content, err := scenario.fromFileBox(thisFile)
+		if err != nil {
+			err := fmt.Errorf("error reading %q: %w", thisFile, err)
+			trace.Trace(err)
+			return fileTypes, err
+		}
+		fileTypes, err = yamlDocTypes(content)
+		if err != nil {
+			return fileTypes, err
+		}
+	}
+	fileTypes = append(fileTypes, "scenario")
+	return fileTypes, err
+}
+
 func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 	type docSchema struct {
 		Type string
@@ -173,14 +209,14 @@ func (scenario *Scenario) createSandbox() (err error) {
 		return err
 	}
 
-	includePaths, err := scenario.copyFiles(scenario.files, true)
+	err = scenario.copyFiles(scenario.filesByType["scenario"])
 	if err != nil {
 		err = fmt.Errorf("could not copy scenario files: %w", err)
 		trace.Trace(err)
 		return err
 	}
 
-	if err := scenario.getIncludes(includePaths); err != nil {
+	if err = scenario.getIncludes(scenario.filesByType["include"]); err != nil {
 		err = fmt.Errorf("could not copy included schema files: %w", err)
 		trace.Trace(err)
 		return err
@@ -190,11 +226,19 @@ func (scenario *Scenario) createSandbox() (err error) {
 	return nil
 }
 
-const includeFilePrefix = "include."
-
 func (scenario *Scenario) getIncludes(includes []string) error {
 	for _, includeFile := range includes {
-		dstPath := includeFile[len(includeFilePrefix):]
+
+		// The following guards against includeFilePrefix being present elsewhere in includeFile. Otherwise, we could use `dstPath := strings.Replace(includeFile, includeFilePrefix, "")`
+		prefixStartIdx := strings.LastIndex(includeFile, includeFilePrefix)
+		prefixEndIdx := prefixStartIdx + len(includeFilePrefix)
+		if prefixStartIdx < 0 || prefixEndIdx >= len(includeFile) {
+			msg := fmt.Sprintf("logic error: start %d, end %d outside of range [0, %d] for %q", prefixStartIdx, prefixEndIdx, len(includeFile), includeFile)
+			trace.Trace(msg)
+			panic(msg)
+		}
+		dstPath := includeFile[0:prefixStartIdx] + includeFile[prefixEndIdx:]
+
 		content, err := scenario.fromFileBox(includeFile)
 		if err != nil {
 			err = fmt.Errorf("could not read %q: %w", includeFile, err)
@@ -202,44 +246,37 @@ func (scenario *Scenario) getIncludes(includes []string) error {
 			return err
 		}
 
-		files, srcPath, replacePath, err := GetMatchingFiles(scenario.schemaBox, dstPath, string(content), '/')
+		srcPath := strings.TrimSpace(string(content))
+		srcPath = strings.ReplaceAll(srcPath, string(includeFileDirectorySeparator), string(os.PathSeparator))
+		files, replacePath, err := GetMatchingFiles(scenario.schemaBox, dstPath, srcPath)
 		if err != nil {
 			err = fmt.Errorf("could not process %q: %w", includeFile, err)
 			trace.Trace(err)
 			return err
 		}
 
-		scenario.copyFilesReplacePath(files, false, scenario.schemaBox.Find, srcPath, replacePath)
+		scenario.copyFilesReplacePath(files, scenario.schemaBox.Find, srcPath, replacePath)
 	}
 	return nil
 }
 
-func (scenario *Scenario) copyFiles(files []string, returnIncludes bool) (includes []string, err error) {
-	return scenario.copyFilesReplacePath(files, returnIncludes, scenario.fromFileBox, "", "")
+func (scenario *Scenario) copyFiles(files []string) error {
+	return scenario.copyFilesReplacePath(files, scenario.fromFileBox, "", "")
 }
 
-func (scenario *Scenario) copyFilesReplacePath(files []string, returnIncludes bool, fromBox func(string) ([]byte, error), replaceSrc, replaceDst string) (includes []string, err error) {
+func (scenario *Scenario) copyFilesReplacePath(files []string, fromBox func(string) ([]byte, error), replaceSrc, replaceDst string) (err error) {
 	const filePermissions = 0555
 	replace := len(replaceSrc) > 0
 
-	trace.Trace("returnIncludes: %v  replaceSrc:%q  replaceDst:%q  files%q",
-		returnIncludes, replaceSrc, replaceDst, files)
+	trace.Trace("replaceSrc:%q  replaceDst:%q  len(files):%d", replaceSrc, replaceDst, len(files))
 
 	for _, srcFile := range files {
-
-		if returnIncludes && strings.HasPrefix(srcFile, includeFilePrefix) && strings.IndexAny(srcFile, string(os.PathSeparator)) == -1 {
-			// This is an include.* file
-			includes = append(includes, srcFile)
-			continue
-		}
-
 		renamedFile := srcFile
 		if replace {
 			if !strings.HasPrefix(renamedFile, replaceSrc) {
 				panic(fmt.Sprintf("%q does not begin with %q", srcFile, replaceSrc))
 			}
 			renamedFile = strings.Replace(renamedFile, replaceSrc, replaceDst, 1)
-			trace.Trace("renamedFile: %q", renamedFile)
 		}
 
 		renamedDir := filepath.Dir(renamedFile)
@@ -249,7 +286,7 @@ func (scenario *Scenario) copyFilesReplacePath(files []string, returnIncludes bo
 			if err = os.MkdirAll(dstDir, os.ModePerm); err != nil {
 				err = fmt.Errorf("could not make directory %q: %w", dstDir, err)
 				trace.Trace(err)
-				return includes, err
+				return err
 			}
 		}
 
@@ -257,16 +294,15 @@ func (scenario *Scenario) copyFilesReplacePath(files []string, returnIncludes bo
 		if contents, err = fromBox(srcFile); err != nil {
 			err = fmt.Errorf("could not find file %q: %w", srcFile, err)
 			trace.Trace(err)
-			return includes, err
+			return err
 		}
 
 		dstFile := scenario.sandboxPath(renamedFile)
 		if err = ioutil.WriteFile(dstFile, contents, filePermissions); err != nil {
 			err = fmt.Errorf("could not write file  %q: %w", dstFile, err)
 			trace.Trace(err)
-			return includes, err
+			return err
 		}
-		trace.Trace(dstFile)
 	}
-	return includes, nil
+	return nil
 }

From 13895993da416051c8928274913f791b7b599da7 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Tue, 8 Oct 2019 15:39:00 -0700
Subject: [PATCH 22/39] Better trace the assets read in

---
 cmd/qualify/assets.go | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index 4aec55a1c..e101eb647 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -13,15 +13,18 @@ var AcceptanceSuite *packr.Box
 var SchemaSuite *packr.Box
 
 func GetAssets() {
+	// I believe we can't pass the arguments into a function
+	// because otherwise packr won't be able to recognize these
+	// paths should be packed.
 	AcceptanceSuite = packr.New("acceptance suite", "../../acceptance")
-	trace.Trace("packr suite: %v", AcceptanceSuite)
-	contents := AcceptanceSuite.List()
-	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
-
 	SchemaSuite = packr.New("schema", "../../schema")
-	trace.Trace("packr suite: %v", SchemaSuite)
-	contents = SchemaSuite.List()
-	trace.Trace("in the suite: \n  %v", strings.Join(contents, "\n  "))
+
+	traceBox(AcceptanceSuite)
+	traceBox(SchemaSuite)
+}
+
+func traceBox(box *packr.Box) {
+	trace.Trace("suite %q has %d entries", box.Name, len(box.List()))
 }
 
 type FilesByDir struct {

From 2acf5c1ae48491cff59706c85a454ed5d28671d9 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Wed, 9 Oct 2019 11:46:11 -0700
Subject: [PATCH 23/39] Check the generation step succeded via zero exit code

In the future, we'll be able to configure expected failures via a
config file, and check for those as well.

This also includes miscellaneous documentation improvements.
---
 cmd/qualify/generator.go | 16 ++++++--
 cmd/qualify/scenario.go  | 83 +++++++++++++++++++++-------------------
 2 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
index 75369c4d9..57628434a 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/qualify/generator.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -17,14 +18,14 @@ type GeneratorInfo struct {
 	isMonolith bool
 }
 
-func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]byte, error) {
+func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
 	generationDir := "generated"
 	if gi.isMonolith {
-		return nil, fmt.Errorf("monolith not implemented yet")
+		return nil, nil, fmt.Errorf("monolith not implemented yet")
 	}
 
 	if err := os.MkdirAll(path.Join(wdir, generationDir), os.ModePerm); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	cmdParts := []string{
@@ -39,5 +40,12 @@ func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]by
 	trace.Trace("running: %s", cmdString)
 	cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
 	cmd.Dir = wdir
-	return cmd.CombinedOutput()
+	output, err := cmd.CombinedOutput()
+
+	var exitError *exec.ExitError
+	if errors.As(err, &exitError) {
+		err = nil
+		trace.Trace("clearing exit error: %v", exitError)
+	}
+	return output, cmd.ProcessState, err
 }
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 6ad5ab8c5..24d5b7772 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -38,15 +38,19 @@ type Scenario struct {
 
 	generator    *GeneratorInfo
 	showcasePort int
+
+	generationOutput       []byte
+	generationProcessState *os.ProcessState
+	generationPassed       bool
 }
 
 func (scenario *Scenario) sandboxPath(relativePath string) string {
 	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q!", relativePath)
+		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
 		panic("scenario name not set yet when trying to get sandbox filename")
 	}
 	if len(scenario.sandbox) == 0 {
-		trace.Trace("sandbox name not set yet: relativePath==%q!", relativePath)
+		trace.Trace("sandbox name not set yet: relativePath==%q", relativePath)
 		panic("sandbox name not set yet when trying to get sandbox filename")
 	}
 
@@ -55,7 +59,7 @@ func (scenario *Scenario) sandboxPath(relativePath string) string {
 
 func (scenario *Scenario) fileBoxPath(relativePath string) string {
 	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q!", relativePath)
+		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
 		panic("scenario name not set yet when trying to get box filename")
 	}
 	return filepath.Join(scenario.name, relativePath)
@@ -66,6 +70,15 @@ func (scenario *Scenario) fromFileBox(relativePath string) ([]byte, error) {
 }
 
 func (scenario *Scenario) Run() error {
+	if err := scenario.classifyConfigs(); err != nil {
+		return fmt.Errorf("could not classify config files for scenario %q: %w", scenario.name, err)
+	}
+
+	if err := scenario.createSandbox(); err != nil {
+		return fmt.Errorf("could not create sandbox for scenario %q: %w", scenario.name, err)
+	}
+	trace.Trace("created sandbox: %q", scenario.sandbox)
+
 	if err := scenario.Generate(); err != nil {
 		return err
 	}
@@ -74,34 +87,18 @@ func (scenario *Scenario) Run() error {
 	return nil
 }
 
-func (scenario *Scenario) Generate() (err error) {
-	if err = scenario.getGenerationFiles(); err != nil {
-		return err
-	}
-	err = scenario.createSandbox()
-	if err != nil {
-		return fmt.Errorf("could not create sandbox: %w", err)
-	}
-	trace.Trace("created sandbox: %q", scenario.sandbox)
-
-	var output []byte
-	output, err = scenario.generator.Run(scenario.sandbox, scenario.filesByType)
+func (scenario *Scenario) Generate() error {
+	var err error
+	scenario.generationOutput, scenario.generationProcessState, err = scenario.generator.Run(scenario.sandbox, scenario.filesByType)
 	trace.Trace("run error: %v", err)
-	trace.Trace("run output: %s", output)
-
-	/* TODO:
-	   - P1. Invoke sample generator with all the protos in the directory
-	   - P2. when creating sandbox, expand files of the form `include.FOO` to be file/dir FOO/ with contents cloned from the location specified in the file
-	        can be a path rooted at schema/
-	        can be an http: which is curled
-	*/
-	trace.Trace("Generate (TODO)")
-	return nil
+	trace.Trace("run scenario.generationOutput: %s", scenario.generationOutput)
+	return err
 }
 
 func (scenario *Scenario) CheckGeneration() {
-	// TODO: Fill in
-	trace.Trace("CheckGeneration (TODO)")
+	// TODO: Fill in in order to handle expected, configured failures
+	trace.Trace("CheckGeneration")
+	scenario.generationPassed = scenario.generationProcessState.Success()
 }
 
 func (scenario *Scenario) RunTests() {
@@ -114,13 +111,11 @@ func (scenario *Scenario) Success() bool {
 	return true
 }
 
-// getGenerationFiles classifies all the files in the scenario,
-// storing the results in `scenario.filesByType`. Each file is
-// classified as either an "include" file or a "scenario"
-// file. Scenario files are also classified by the type of data they
-// contain. Thus, "scenario" files have at least two labels (ie
-// multiple entries in `scenario.filesByType`).
-func (scenario *Scenario) getGenerationFiles() (err error) {
+// classifyConfigs classifies all the files in the scenario, storing the results in
+// `scenario.filesByType`. Each file is classified as either an "include" file or a "scenario"
+// file. Scenario files are also classified by the type of data they contain. Thus, "scenario" files
+// have at least two labels (ie multiple entries in `scenario.filesByType`).
+func (scenario *Scenario) classifyConfigs() (err error) {
 	scenario.filesByType = make(map[string][]string)
 	// TODO: process `include.*` files first
 	for _, thisFile := range scenario.files {
@@ -137,10 +132,9 @@ func (scenario *Scenario) getGenerationFiles() (err error) {
 	return nil
 }
 
-// getFileTypes gets the various type labels for `thisFile`. The type
-// labels are either the singleton list {"include"}, or a list
-// {"scenario", ...} that includes the various types of data included
-// in that file.
+// getFileTypes gets the various type labels for `thisFile`. The type labels are either the
+// singleton list {"include"}, or a list {"scenario", ...} that includes the various types of data
+// included in that file.
 func (scenario *Scenario) getFileTypes(thisFile string) (fileTypes []string, err error) {
 
 	for idx := strings.Index(thisFile, includeFilePrefix); idx >= 0; idx = strings.Index(thisFile[idx+1:], includeFilePrefix) {
@@ -226,10 +220,21 @@ func (scenario *Scenario) createSandbox() (err error) {
 	return nil
 }
 
+// getIncludes processes include files (whose names start with `includeFilePrefix`) by replacing
+// them with the file or directory referenced in the file's contents. The only currently supported
+// type of reference is to either an entry in `scenario.schemaBox`, or a directory that resolves to
+// one or more entries in `scenario.schemaBox`. If an include file does not resolve to any entries
+// in `scenario.schemaBox`, this function returns an error.
+//
+// If there's need in the future, we could potentially grow to support fetching files or directories
+// from public repositories on the web.
 func (scenario *Scenario) getIncludes(includes []string) error {
 	for _, includeFile := range includes {
 
-		// The following guards against includeFilePrefix being present elsewhere in includeFile. Otherwise, we could use `dstPath := strings.Replace(includeFile, includeFilePrefix, "")`
+		// The following guards against includeFilePrefix being present elsewhere in
+		// includeFile. Otherwise, we could simply use
+		//
+		//   `dstPath := strings.Replace(includeFile, includeFilePrefix, "")`
 		prefixStartIdx := strings.LastIndex(includeFile, includeFilePrefix)
 		prefixEndIdx := prefixStartIdx + len(includeFilePrefix)
 		if prefixStartIdx < 0 || prefixEndIdx >= len(includeFile) {

From 238659cf57e5dd5534e2bd5308b5526adf5f5f3e Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Thu, 10 Oct 2019 17:28:32 -0700
Subject: [PATCH 24/39] Make the generation step run with samplegen turned on

In the process, constructed a sample config for the echo service and
made the hard-coded generator default be protoc-gen-python_gapic via protoc.
---
 acceptance/passing/sample.yaml | 19 +++++++++++++++++--
 cmd/qualify/assets.go          |  3 +++
 cmd/qualify/generator.go       | 34 +++++++++++++++++++++++++++-------
 cmd/qualify/qualify.go         | 18 +++++++++++++++---
 cmd/qualify/scenario.go        | 10 ++++++----
 5 files changed, 68 insertions(+), 16 deletions(-)

diff --git a/acceptance/passing/sample.yaml b/acceptance/passing/sample.yaml
index c4a2e3ad0..d1b08fca5 100644
--- a/acceptance/passing/sample.yaml
+++ b/acceptance/passing/sample.yaml
@@ -1,6 +1,21 @@
 ---
-type: sampleconfig
-schema_version: 2
+type: com.google.api.codegen.SampleConfigProto
+schema_version: 1.2.0
+samples:
+- service: google.showcase.v1beta1.Echo
+  rpc: Echo
+  description: A sample of a simple echo request and response
+  request:
+  - field: content
+    comment: The content to be echoed back
+    value: a resounding echo...echo...echo
+    input_parameter: message
+  response:
+    - comment:
+      - "We simply print the response"
+    - print:
+      - 'Heard back: "%s"'
+      - $resp.content
 ---
 type: test/sample
 schema_version: 1
diff --git a/cmd/qualify/assets.go b/cmd/qualify/assets.go
index e101eb647..50706a1dd 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/qualify/assets.go
@@ -12,6 +12,9 @@ import (
 var AcceptanceSuite *packr.Box
 var SchemaSuite *packr.Box
 
+const fileTypeSampleConfig = "com.google.api.codegen.SampleConfigProto"
+const fileTypeProtobuf = "proto"
+
 func GetAssets() {
 	// I believe we can't pass the arguments into a function
 	// because otherwise packr won't be able to recognize these
diff --git a/cmd/qualify/generator.go b/cmd/qualify/generator.go
index 57628434a..4fc06f9d0 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/qualify/generator.go
@@ -11,6 +11,9 @@ import (
 	trace "github.com/google/go-trace"
 )
 
+// GeneratorInfo has the information needed to run a generator in a given language. The generator
+// will be run as a protoc plugin unless `isMonolith` is set, in which case gapic-generator will be
+// invoked directly.
 type GeneratorInfo struct {
 	name       string
 	dir        string
@@ -18,28 +21,45 @@ type GeneratorInfo struct {
 	isMonolith bool
 }
 
-func (gi *GeneratorInfo) Run(wdir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
+// Run will run `gi` from `work_dir`, obtaining the required files from filesByType.
+func (gi *GeneratorInfo) Run(workDir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
 	generationDir := "generated"
 	if gi.isMonolith {
 		return nil, nil, fmt.Errorf("monolith not implemented yet")
 	}
 
-	if err := os.MkdirAll(path.Join(wdir, generationDir), os.ModePerm); err != nil {
+	if err := os.MkdirAll(path.Join(workDir, generationDir), os.ModePerm); err != nil {
 		return nil, nil, err
 	}
 
+	optionFlag := fmt.Sprintf("--%s_gapic_opt", gi.name)
+	allOptions := []string{}
+	if len(gi.options) > 0 {
+		allOptions = append(allOptions, optionFlag, gi.options)
+	}
+
+	sampleConfigFiles, _ := filesByType[fileTypeSampleConfig]
+	if len(sampleConfigFiles) > 0 {
+		allOptions = append(allOptions, optionFlag,
+			fmt.Sprintf("samples=%s", strings.Join(sampleConfigFiles, ",samples=")))
+	}
+
 	cmdParts := []string{
 		"protoc",
 		fmt.Sprintf("--%s_gapic_out", gi.name), fmt.Sprintf("./%s", generationDir),
-		fmt.Sprintf("--%s_gapic_opt", gi.name), gi.options,
-		fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name),
 	}
-	cmdParts = append(cmdParts, filesByType["proto"]...)
-	cmdString := strings.Join(cmdParts, " ")
+	cmdParts = append(cmdParts, allOptions...)
+	if len(gi.dir) > 0 {
+		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name))
 
+	}
+	cmdParts = append(cmdParts, filesByType[fileTypeProtobuf]...)
+
+	cmdString := strings.Join(cmdParts, " ")
 	trace.Trace("running: %s", cmdString)
+
 	cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
-	cmd.Dir = wdir
+	cmd.Dir = workDir
 	output, err := cmd.CombinedOutput()
 
 	var exitError *exec.ExitError
diff --git a/cmd/qualify/qualify.go b/cmd/qualify/qualify.go
index 58c9f2d90..77baf062a 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/qualify/qualify.go
@@ -69,9 +69,12 @@ func main() {
 		if err := scenario.Run(); err != nil {
 			os.Exit(RetCodeInternalError)
 		}
+		status := "SUCCESS"
 		if !scenario.Success() {
 			success = false
+			status = "FAILURE"
 		}
+		fmt.Printf("scenario %q: %s    %s\n", scenario.name, status, scenario.sandbox)
 	}
 	if !success {
 		os.Exit(RetScenarioFailure)
@@ -108,9 +111,18 @@ func checkDependencies() error {
 // line, and whether it is a protoc plugin (if not, it is considered
 // part of the monolith)
 func getGeneratorData() (*GeneratorInfo, error) {
-	pluginName := "go"                                                               // TODO: get from CLI args
-	pluginDir := "/tmp/goinstall/bin"                                                // TODO: get from CLI args
-	pluginOpt := "go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase" // TODO: get from CLI args
+	// pluginName := "go"                                                               // TODO: get from CLI args
+	// pluginDir := "/tmp/goinstall/bin"                                                // TODO: get from CLI args
+	// pluginOpt := "go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase" // TODO: get from CLI args
+	// generator := &GeneratorInfo{
+	// 	name:    pluginName,
+	// 	dir:     pluginDir,
+	// 	options: pluginOpt,
+	// }
+
+	pluginName := "python"
+	pluginDir := ""
+	pluginOpt := ""
 	generator := &GeneratorInfo{
 		name:    pluginName,
 		dir:     pluginDir,
diff --git a/cmd/qualify/scenario.go b/cmd/qualify/scenario.go
index 24d5b7772..27a23d7ef 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/qualify/scenario.go
@@ -26,9 +26,9 @@ func init() {
 }
 
 type Scenario struct {
-	// Every f in `files` corresponds to a file in `fileBox` under
+	// Every element in `files` corresponds to a file in `fileBox` under
 	// a directory called `name`, which is NOT part of the string
-	// f itself.
+	// element itself.
 	name        string
 	files       []string
 	fileBox     *packr.Box
@@ -42,6 +42,7 @@ type Scenario struct {
 	generationOutput       []byte
 	generationProcessState *os.ProcessState
 	generationPassed       bool
+	sampleTestsPassed      bool
 }
 
 func (scenario *Scenario) sandboxPath(relativePath string) string {
@@ -107,8 +108,9 @@ func (scenario *Scenario) RunTests() {
 }
 
 func (scenario *Scenario) Success() bool {
-	// TODO: fill in
-	return true
+	return scenario.generationPassed && // the generation checks passed and...
+		(!scenario.generationProcessState.Success() || // ...either generation failed, or it succeeded with ...
+			(len(scenario.filesByType["test/samples"]) == 0 || scenario.sampleTestsPassed)) // all sample tests, if any, passing
 }
 
 // classifyConfigs classifies all the files in the scenario, storing the results in

From 27f00d9d74744da3c269fe4a1b6c2659323f9ae9 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 11 Oct 2019 11:14:46 -0700
Subject: [PATCH 25/39] Move qualify to be a gapic-showcase subcommand

This creates a package `qualifier`
---
 .../acceptance_suite}/passing/echo.proto      |  0
 .../passing/foo/bar/include.wdREAD            |  0
 .../acceptance_suite}/passing/include.google  |  0
 .../acceptance_suite}/passing/include.vcREAD  |  0
 .../acceptance_suite}/passing/sample.yaml     |  0
 .../qualifier}/assets.go                      |  6 ++---
 .../qualifier}/generator.go                   |  2 +-
 .../qualifier}/prepare-to-qualify             |  2 +-
 .../qualifier}/qualify.go                     |  5 +++--
 .../qualifier}/scenario.go                    |  2 +-
 cmd/gapic-showcase/qualify.go                 | 22 +++++++++++++++++++
 11 files changed, 31 insertions(+), 8 deletions(-)
 rename {acceptance => cmd/gapic-showcase/qualifier/acceptance_suite}/passing/echo.proto (100%)
 rename {acceptance => cmd/gapic-showcase/qualifier/acceptance_suite}/passing/foo/bar/include.wdREAD (100%)
 rename {acceptance => cmd/gapic-showcase/qualifier/acceptance_suite}/passing/include.google (100%)
 rename {acceptance => cmd/gapic-showcase/qualifier/acceptance_suite}/passing/include.vcREAD (100%)
 rename {acceptance => cmd/gapic-showcase/qualifier/acceptance_suite}/passing/sample.yaml (100%)
 rename cmd/{qualify => gapic-showcase/qualifier}/assets.go (95%)
 rename cmd/{qualify => gapic-showcase/qualifier}/generator.go (99%)
 rename cmd/{qualify => gapic-showcase/qualifier}/prepare-to-qualify (97%)
 rename cmd/{qualify => gapic-showcase/qualifier}/qualify.go (98%)
 rename cmd/{qualify => gapic-showcase/qualifier}/scenario.go (99%)
 create mode 100644 cmd/gapic-showcase/qualify.go

diff --git a/acceptance/passing/echo.proto b/cmd/gapic-showcase/qualifier/acceptance_suite/passing/echo.proto
similarity index 100%
rename from acceptance/passing/echo.proto
rename to cmd/gapic-showcase/qualifier/acceptance_suite/passing/echo.proto
diff --git a/acceptance/passing/foo/bar/include.wdREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/passing/foo/bar/include.wdREAD
similarity index 100%
rename from acceptance/passing/foo/bar/include.wdREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/passing/foo/bar/include.wdREAD
diff --git a/acceptance/passing/include.google b/cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.google
similarity index 100%
rename from acceptance/passing/include.google
rename to cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.google
diff --git a/acceptance/passing/include.vcREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.vcREAD
similarity index 100%
rename from acceptance/passing/include.vcREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.vcREAD
diff --git a/acceptance/passing/sample.yaml b/cmd/gapic-showcase/qualifier/acceptance_suite/passing/sample.yaml
similarity index 100%
rename from acceptance/passing/sample.yaml
rename to cmd/gapic-showcase/qualifier/acceptance_suite/passing/sample.yaml
diff --git a/cmd/qualify/assets.go b/cmd/gapic-showcase/qualifier/assets.go
similarity index 95%
rename from cmd/qualify/assets.go
rename to cmd/gapic-showcase/qualifier/assets.go
index 50706a1dd..f3d52f4bb 100644
--- a/cmd/qualify/assets.go
+++ b/cmd/gapic-showcase/qualifier/assets.go
@@ -1,4 +1,4 @@
-package main
+package qualifier
 
 import (
 	"fmt"
@@ -19,8 +19,8 @@ func GetAssets() {
 	// I believe we can't pass the arguments into a function
 	// because otherwise packr won't be able to recognize these
 	// paths should be packed.
-	AcceptanceSuite = packr.New("acceptance suite", "../../acceptance")
-	SchemaSuite = packr.New("schema", "../../schema")
+	AcceptanceSuite = packr.New("acceptance suite", "acceptance_suite")
+	SchemaSuite = packr.New("schema", "../../../schema")
 
 	traceBox(AcceptanceSuite)
 	traceBox(SchemaSuite)
diff --git a/cmd/qualify/generator.go b/cmd/gapic-showcase/qualifier/generator.go
similarity index 99%
rename from cmd/qualify/generator.go
rename to cmd/gapic-showcase/qualifier/generator.go
index 4fc06f9d0..af53635d9 100644
--- a/cmd/qualify/generator.go
+++ b/cmd/gapic-showcase/qualifier/generator.go
@@ -1,4 +1,4 @@
-package main
+package qualifier
 
 import (
 	"errors"
diff --git a/cmd/qualify/prepare-to-qualify b/cmd/gapic-showcase/qualifier/prepare-to-qualify
similarity index 97%
rename from cmd/qualify/prepare-to-qualify
rename to cmd/gapic-showcase/qualifier/prepare-to-qualify
index 53842a18c..06d8aad3a 100644
--- a/cmd/qualify/prepare-to-qualify
+++ b/cmd/gapic-showcase/qualifier/prepare-to-qualify
@@ -3,7 +3,7 @@ VENV_NAME=venv/qualify
 DATE=$(date +'%Y-%m-%d.%H-%M')
 
 BUILD_DIR=$(pwd)
-go build ../gapic-showcase/
+go build ../../gapic-showcase/
 
 verify() {
   name=$1
diff --git a/cmd/qualify/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
similarity index 98%
rename from cmd/qualify/qualify.go
rename to cmd/gapic-showcase/qualifier/qualify.go
index 77baf062a..e44a71ff3 100644
--- a/cmd/qualify/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -1,4 +1,4 @@
-package main
+package qualifier
 
 import (
 	"fmt"
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	trace "github.com/google/go-trace"
+	"github.com/spf13/cobra"
 )
 
 /* Development instructions
@@ -28,7 +29,7 @@ Choosing: https://tech.townsourced.com/post/embedding-static-files-in-go/
 
 const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
-func main() {
+func RunProcess(cmd *cobra.Command, args []string) {
 	const (
 		RetCodeSuccess = iota
 		RetCodeInternalError
diff --git a/cmd/qualify/scenario.go b/cmd/gapic-showcase/qualifier/scenario.go
similarity index 99%
rename from cmd/qualify/scenario.go
rename to cmd/gapic-showcase/qualifier/scenario.go
index 27a23d7ef..8ee959a4b 100644
--- a/cmd/qualify/scenario.go
+++ b/cmd/gapic-showcase/qualifier/scenario.go
@@ -1,4 +1,4 @@
-package main
+package qualifier
 
 import (
 	"bytes"
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
new file mode 100644
index 000000000..85d88227c
--- /dev/null
+++ b/cmd/gapic-showcase/qualify.go
@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/googleapis/gapic-showcase/cmd/gapic-showcase/qualifier"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	qualifyCmd := &cobra.Command{
+		Use:   "qualify",
+		Short: "Tests a provided gapic generator against an acceptance suite",
+		Run:   qualifier.RunProcess,
+	}
+	rootCmd.AddCommand(qualifyCmd)
+}
+
+func qualifyStub(cmd *cobra.Command, args []string) {
+	fmt.Printf("in qualify!\n")
+}

From b7732227062f1f8c4213ba7e9d00294d28d49acf Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 11 Oct 2019 11:31:08 -0700
Subject: [PATCH 26/39] Remove debugging comments

---
 cmd/gapic-showcase/qualifier/prepare-to-qualify | 7 -------
 cmd/gapic-showcase/qualifier/qualify.go         | 8 +-------
 cmd/gapic-showcase/qualify.go                   | 2 +-
 3 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/prepare-to-qualify b/cmd/gapic-showcase/qualifier/prepare-to-qualify
index 06d8aad3a..9ea0fe366 100644
--- a/cmd/gapic-showcase/qualifier/prepare-to-qualify
+++ b/cmd/gapic-showcase/qualifier/prepare-to-qualify
@@ -36,10 +36,3 @@ echo '=== creating venv'  && \
 popd >& /dev/null
 echo -e "\nInstalled in: ${GAPIC_QUALIFY_DIR}"
 
-# To get the Go plugin to work:
-#
-# VC_PROTO_PATH=/home/vchudnov/Work/ACTools/gapic-showcase/acceptance/passing
-# GO_PLUGIN_DIR=/tmp/goinstall
-# mkdir ${GO_PLUGIN_DIR}
-# GOPATH=${GO_PLUGIN_DIR} go install github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
-# protoc --go_gapic_out ./gen --go_gapic_opt 'go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase' --plugin=${GO_PLUGIN_DIR}/bin/protoc-gen-go_gapic --proto_path=${VC_PROTO_PATH} ${VC_PROTO_PATH}/echo.proto
diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index e44a71ff3..2ec0c55b5 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -18,18 +18,12 @@ import (
 === Sample run ======
 go run *.go
 
-Notes for upcoming development:
-packing assets into Go files:
-https://www.google.com/search?q=golang+ship+data+files+with+binary&oq=golang+ship+data+files+with+binary
-https://github.com/go-bindata/go-bindata
-https://github.com/gobuffalo/packr
-Choosing: https://tech.townsourced.com/post/embedding-static-files-in-go/
 
 */
 
 const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
-func RunProcess(cmd *cobra.Command, args []string) {
+func Run(cmd *cobra.Command, args []string) {
 	const (
 		RetCodeSuccess = iota
 		RetCodeInternalError
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index 85d88227c..04f127634 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -12,7 +12,7 @@ func init() {
 	qualifyCmd := &cobra.Command{
 		Use:   "qualify",
 		Short: "Tests a provided gapic generator against an acceptance suite",
-		Run:   qualifier.RunProcess,
+		Run:   qualifier.Run,
 	}
 	rootCmd.AddCommand(qualifyCmd)
 }

From 8fbe2382ae965bb864123edf26d2c7c217aa74c0 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 11 Oct 2019 12:58:51 -0700
Subject: [PATCH 27/39] Allow passing flags and args to the qualify command

---
 cmd/gapic-showcase/qualifier/generator.go | 30 +++++++-------
 cmd/gapic-showcase/qualifier/qualify.go   | 50 +++++------------------
 cmd/gapic-showcase/qualifier/scenario.go  | 13 ++----
 cmd/gapic-showcase/qualify.go             | 41 +++++++++++++++----
 4 files changed, 62 insertions(+), 72 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/generator.go b/cmd/gapic-showcase/qualifier/generator.go
index af53635d9..69da1a4f4 100644
--- a/cmd/gapic-showcase/qualifier/generator.go
+++ b/cmd/gapic-showcase/qualifier/generator.go
@@ -11,20 +11,20 @@ import (
 	trace "github.com/google/go-trace"
 )
 
-// GeneratorInfo has the information needed to run a generator in a given language. The generator
+// Generator has the information needed to run a generator in a given language. The generator
 // will be run as a protoc plugin unless `isMonolith` is set, in which case gapic-generator will be
 // invoked directly.
-type GeneratorInfo struct {
-	name       string
-	dir        string
-	options    string
-	isMonolith bool
+type Generator struct {
+	Language        string
+	PluginDirectory string
+	PluginOptions   string
+	isMonolith      bool
 }
 
-// Run will run `gi` from `work_dir`, obtaining the required files from filesByType.
-func (gi *GeneratorInfo) Run(workDir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
+// Run will run the generator from `work_dir`, obtaining the required files from filesByType.
+func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
 	generationDir := "generated"
-	if gi.isMonolith {
+	if gen.isMonolith {
 		return nil, nil, fmt.Errorf("monolith not implemented yet")
 	}
 
@@ -32,10 +32,10 @@ func (gi *GeneratorInfo) Run(workDir string, filesByType map[string][]string) ([
 		return nil, nil, err
 	}
 
-	optionFlag := fmt.Sprintf("--%s_gapic_opt", gi.name)
+	optionFlag := fmt.Sprintf("--%s_gapic_opt", gen.Language)
 	allOptions := []string{}
-	if len(gi.options) > 0 {
-		allOptions = append(allOptions, optionFlag, gi.options)
+	if len(gen.PluginOptions) > 0 {
+		allOptions = append(allOptions, optionFlag, gen.PluginOptions)
 	}
 
 	sampleConfigFiles, _ := filesByType[fileTypeSampleConfig]
@@ -46,11 +46,11 @@ func (gi *GeneratorInfo) Run(workDir string, filesByType map[string][]string) ([
 
 	cmdParts := []string{
 		"protoc",
-		fmt.Sprintf("--%s_gapic_out", gi.name), fmt.Sprintf("./%s", generationDir),
+		fmt.Sprintf("--%s_gapic_out", gen.Language), fmt.Sprintf("./%s", generationDir),
 	}
 	cmdParts = append(cmdParts, allOptions...)
-	if len(gi.dir) > 0 {
-		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gi.dir, gi.name))
+	if len(gen.PluginDirectory) > 0 {
+		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gen.PluginDirectory, gen.Language))
 
 	}
 	cmdParts = append(cmdParts, filesByType[fileTypeProtobuf]...)
diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index 2ec0c55b5..229317b40 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 
 	trace "github.com/google/go-trace"
-	"github.com/spf13/cobra"
 )
 
 /* Development instructions
@@ -21,9 +20,15 @@ go run *.go
 
 */
 
+type Settings struct {
+	Generator
+	Timestamp string
+	Verbose   bool
+}
+
 const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
-func Run(cmd *cobra.Command, args []string) {
+func Run(settings *Settings) {
 	const (
 		RetCodeSuccess = iota
 		RetCodeInternalError
@@ -33,10 +38,6 @@ func Run(cmd *cobra.Command, args []string) {
 		RetScenarioFailure
 	)
 
-	debugMe := true // TODO: get from CLI args
-	trace.On(debugMe)
-	trace.Trace("timestamp: %s", timestamp)
-
 	GetAssets()
 
 	if err := checkDependencies(); err != nil {
@@ -49,12 +50,7 @@ func Run(cmd *cobra.Command, args []string) {
 	}
 	defer endProcess(showcase)
 
-	generatorData, err := getGeneratorData()
-	if err != nil {
-		os.Exit(RetCodeUsageError)
-	}
-
-	allScenarios, err := GetTestScenarios(generatorData)
+	allScenarios, err := GetTestScenarios(settings)
 	if err != nil {
 		os.Exit(RetCodeInternalError)
 	}
@@ -102,42 +98,18 @@ func checkDependencies() error {
 	return nil
 }
 
-// getGeneratorData obtains the name of the generator from the command
-// line, and whether it is a protoc plugin (if not, it is considered
-// part of the monolith)
-func getGeneratorData() (*GeneratorInfo, error) {
-	// pluginName := "go"                                                               // TODO: get from CLI args
-	// pluginDir := "/tmp/goinstall/bin"                                                // TODO: get from CLI args
-	// pluginOpt := "go-gapic-package=cloud.google.com/go/showcase/apiv1beta1;showcase" // TODO: get from CLI args
-	// generator := &GeneratorInfo{
-	// 	name:    pluginName,
-	// 	dir:     pluginDir,
-	// 	options: pluginOpt,
-	// }
-
-	pluginName := "python"
-	pluginDir := ""
-	pluginOpt := ""
-	generator := &GeneratorInfo{
-		name:    pluginName,
-		dir:     pluginDir,
-		options: pluginOpt,
-	}
-	trace.Trace("hard-coded generator=%#v", generator)
-	return generator, nil
-}
-
 // GetTestScenarios returns a list of Scenario as found in the specified
 // scenario root directory.
-func GetTestScenarios(generator *GeneratorInfo) ([]*Scenario, error) {
+func GetTestScenarios(settings *Settings) ([]*Scenario, error) {
 	allScenarios := []*Scenario{}
 	allScenarioConfigs := GetFilesByDir(AcceptanceSuite)
 	for _, config := range allScenarioConfigs {
 		defaultShowcasePort := 123 // TODO fix
 		newScenario := &Scenario{
 			name:         config.Directory,
+			timestamp:    settings.Timestamp,
 			showcasePort: defaultShowcasePort,
-			generator:    generator,
+			generator:    &settings.Generator,
 			files:        config.Files,
 			fileBox:      AcceptanceSuite,
 			schemaBox:    SchemaSuite,
diff --git a/cmd/gapic-showcase/qualifier/scenario.go b/cmd/gapic-showcase/qualifier/scenario.go
index 8ee959a4b..1ce920b6e 100644
--- a/cmd/gapic-showcase/qualifier/scenario.go
+++ b/cmd/gapic-showcase/qualifier/scenario.go
@@ -8,35 +8,28 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	packr "github.com/gobuffalo/packr/v2"
 	trace "github.com/google/go-trace"
 	yaml "gopkg.in/yaml.v2"
 )
 
-var timestamp string
-
 const includeFilePrefix = "include."
 const includeFileDirectorySeparator = '/'
 
-func init() {
-	timestamp = time.Now().Format("20060102.150405")
-	trace.Trace("timestamp = %q", timestamp)
-}
-
 type Scenario struct {
 	// Every element in `files` corresponds to a file in `fileBox` under
 	// a directory called `name`, which is NOT part of the string
 	// element itself.
 	name        string
+	timestamp   string
 	files       []string
 	fileBox     *packr.Box
 	schemaBox   *packr.Box
 	sandbox     string
 	filesByType map[string][]string
 
-	generator    *GeneratorInfo
+	generator    *Generator
 	showcasePort int
 
 	generationOutput       []byte
@@ -199,7 +192,7 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 
 func (scenario *Scenario) createSandbox() (err error) {
 
-	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", timestamp, scenario.name)); err != nil {
+	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", scenario.timestamp, scenario.name)); err != nil {
 		err := fmt.Errorf("could not create sandbox: %w", err)
 		trace.Trace(err)
 		return err
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index 04f127634..2a52ae074 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -1,22 +1,47 @@
 package main
 
 import (
-	"fmt"
-
-	"github.com/googleapis/gapic-showcase/cmd/gapic-showcase/qualifier"
+	"time"
 
+	trace "github.com/google/go-trace"
 	"github.com/spf13/cobra"
+
+	"github.com/googleapis/gapic-showcase/cmd/gapic-showcase/qualifier"
 )
 
 func init() {
+	trace.On(false) // set to true for debugging
+
+	var timestamp string
+	timestamp = time.Now().Format("20060102.150405")
+	trace.Trace("timestamp = %q", timestamp)
+
+	settings := &qualifier.Settings{
+		Timestamp: timestamp,
+		Verbose:   Verbose,
+	}
 	qualifyCmd := &cobra.Command{
 		Use:   "qualify",
-		Short: "Tests a provided gapic generator against an acceptance suite",
-		Run:   qualifier.Run,
+		Short: "Tests a provided GAPIC generator against an acceptance suite",
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			settings.Language = args[0]
+			trace.Trace("settings: %v", settings)
+			qualifier.Run(settings)
+		},
 	}
 	rootCmd.AddCommand(qualifyCmd)
-}
+	qualifyCmd.Flags().StringVarP(
+		&settings.PluginDirectory,
+		"dir",
+		"d",
+		"",
+		"The directory in which to find the protoc plugin implementing the given GAPIC generator")
+	qualifyCmd.Flags().StringVarP(
+		&settings.PluginOptions,
+		"options",
+		"o",
+		"",
+		"The options to pass to the protoc plugin in order to generate a GAPIC for the showcase Echo service")
 
-func qualifyStub(cmd *cobra.Command, args []string) {
-	fmt.Printf("in qualify!\n")
 }

From 242179f7f22d5aee272d794c4a0385f9b516dc00 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 11 Oct 2019 14:16:03 -0700
Subject: [PATCH 28/39] Run showcase servers in process when qualifying

This involved refactoring the "run" command so that we can start a
server and not block on it ending.
---
 cmd/gapic-showcase/qualifier/qualify.go |  25 ++---
 cmd/gapic-showcase/qualify.go           |   8 +-
 cmd/gapic-showcase/run.go               | 128 ++++++++++++++++--------
 3 files changed, 98 insertions(+), 63 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index 229317b40..9fa3059ac 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -22,19 +22,22 @@ go run *.go
 
 type Settings struct {
 	Generator
-	Timestamp string
-	Verbose   bool
+	ShowcasePort int
+	Timestamp    string
+	Verbose      bool
 }
 
 const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
 
 func Run(settings *Settings) {
+	// TODO: Return an error rather than exiting. We can return an error type ErrorCode that
+	// wraps the current errors and includes the appropriate return code, and change main() so
+	// that if the error it gets is of type ErrorCode, it exits with that code.
 	const (
 		RetCodeSuccess = iota
 		RetCodeInternalError
 		RetCodeFailedDependencies
 		RetCodeUsageError
-		RetCodeCantRunShowcase
 		RetScenarioFailure
 	)
 
@@ -44,12 +47,6 @@ func Run(settings *Settings) {
 		os.Exit(RetCodeFailedDependencies)
 	}
 
-	showcase, err := startShowcase()
-	if err != nil {
-		os.Exit(RetCodeCantRunShowcase)
-	}
-	defer endProcess(showcase)
-
 	allScenarios, err := GetTestScenarios(settings)
 	if err != nil {
 		os.Exit(RetCodeInternalError)
@@ -77,12 +74,6 @@ func checkDependencies() error {
 	notFound := []string{}
 	trace.Trace("")
 
-	// TODO: Run from this repo, rather than requiring external dependency
-	showcasePath, err := exec.LookPath(showcaseCmd)
-	if err != nil {
-		notFound = append(notFound, showcaseCmd)
-	}
-
 	sampleTesterPath, err := exec.LookPath(sampleTesterCmd)
 	if err != nil {
 		notFound = append(notFound, sampleTesterCmd)
@@ -93,7 +84,6 @@ func checkDependencies() error {
 		log.Printf(msg)
 		return fmt.Errorf(msg)
 	}
-	trace.Trace("found %q: %s", showcaseCmd, showcasePath)
 	trace.Trace("found %q: %s", sampleTesterCmd, sampleTesterPath)
 	return nil
 }
@@ -104,11 +94,10 @@ func GetTestScenarios(settings *Settings) ([]*Scenario, error) {
 	allScenarios := []*Scenario{}
 	allScenarioConfigs := GetFilesByDir(AcceptanceSuite)
 	for _, config := range allScenarioConfigs {
-		defaultShowcasePort := 123 // TODO fix
 		newScenario := &Scenario{
 			name:         config.Directory,
 			timestamp:    settings.Timestamp,
-			showcasePort: defaultShowcasePort,
+			showcasePort: settings.ShowcasePort,
 			generator:    &settings.Generator,
 			files:        config.Files,
 			fileBox:      AcceptanceSuite,
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index 2a52ae074..c7c784f65 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"strconv"
 	"time"
 
 	trace "github.com/google/go-trace"
@@ -17,17 +18,20 @@ func init() {
 	trace.Trace("timestamp = %q", timestamp)
 
 	settings := &qualifier.Settings{
-		Timestamp: timestamp,
-		Verbose:   Verbose,
+		Timestamp:    timestamp,
+		Verbose:      Verbose,
+		ShowcasePort: 7469,
 	}
 	qualifyCmd := &cobra.Command{
 		Use:   "qualify",
 		Short: "Tests a provided GAPIC generator against an acceptance suite",
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			servers := RunShowcase(strconv.Itoa(settings.ShowcasePort), "")
 			settings.Language = args[0]
 			trace.Trace("settings: %v", settings)
 			qualifier.Run(settings)
+			servers.Shutdown()
 		},
 	}
 	rootCmd.AddCommand(qualifyCmd)
diff --git a/cmd/gapic-showcase/run.go b/cmd/gapic-showcase/run.go
index c8452ab9e..019bec266 100644
--- a/cmd/gapic-showcase/run.go
+++ b/cmd/gapic-showcase/run.go
@@ -18,6 +18,7 @@ import (
 	"log"
 	"net"
 	"strings"
+	"sync"
 
 	"github.com/googleapis/gapic-showcase/server"
 	pb "github.com/googleapis/gapic-showcase/server/genproto"
@@ -37,49 +38,7 @@ func init() {
 		Use:   "run",
 		Short: "Runs the showcase server",
 		Run: func(cmd *cobra.Command, args []string) {
-			// Ensure port is of the right form.
-			if !strings.HasPrefix(port, ":") {
-				port = ":" + port
-			}
-
-			// Start listening.
-			lis, err := net.Listen("tcp", port)
-			if err != nil {
-				log.Fatalf("Showcase failed to listen on port '%s': %v", port, err)
-			}
-			stdLog.Printf("Showcase listening on port: %s", port)
-
-			// Setup Server.
-			logger := &loggerObserver{}
-			observerRegistry := server.ShowcaseObserverRegistry()
-			observerRegistry.RegisterUnaryObserver(logger)
-			observerRegistry.RegisterStreamRequestObserver(logger)
-			observerRegistry.RegisterStreamResponseObserver(logger)
-
-			opts := []grpc.ServerOption{
-				grpc.StreamInterceptor(observerRegistry.StreamInterceptor),
-				grpc.UnaryInterceptor(observerRegistry.UnaryInterceptor),
-			}
-			s := grpc.NewServer(opts...)
-			defer s.GracefulStop()
-
-			// Register Services to the server.
-			pb.RegisterEchoServer(s, services.NewEchoServer())
-			identityServer := services.NewIdentityServer()
-			pb.RegisterIdentityServer(s, identityServer)
-			messagingServer := services.NewMessagingServer(identityServer)
-			pb.RegisterMessagingServer(s, messagingServer)
-			operationsServer := services.NewOperationsServer(messagingServer)
-			pb.RegisterTestingServer(s, services.NewTestingServer(observerRegistry))
-			lropb.RegisterOperationsServer(s, operationsServer)
-
-			fb := fallback.NewServer(fallbackPort, "localhost"+port)
-			fb.StartBackground()
-			defer fb.Shutdown()
-
-			// Register reflection service on gRPC server.
-			reflection.Register(s)
-			s.Serve(lis)
+			RunShowcase(port, fallbackPort).Wait()
 		},
 	}
 	rootCmd.AddCommand(runCmd)
@@ -96,3 +55,86 @@ func init() {
 		":1337",
 		"The port that the fallback-proxy will be served on.")
 }
+
+// RunShowcase sets up and starts the showcase and fallback servers and returns pointers to
+// them. They can be shutdown by showcaseServers.Shutdown() or showcaseServers.Wait().
+func RunShowcase(port string, fallbackPort string) (showcaseServers *ShowcaseServers) {
+	// Ensure port is of the right form.
+	if !strings.HasPrefix(port, ":") {
+		port = ":" + port
+	}
+
+	// Start listening.
+	lis, err := net.Listen("tcp", port)
+	if err != nil {
+		log.Fatalf("Showcase failed to listen on port '%s': %v", port, err)
+	}
+	stdLog.Printf("Showcase listening on port: %s", port)
+
+	// Setup Server.
+	logger := &loggerObserver{}
+	observerRegistry := server.ShowcaseObserverRegistry()
+	observerRegistry.RegisterUnaryObserver(logger)
+	observerRegistry.RegisterStreamRequestObserver(logger)
+	observerRegistry.RegisterStreamResponseObserver(logger)
+
+	opts := []grpc.ServerOption{
+		grpc.StreamInterceptor(observerRegistry.StreamInterceptor),
+		grpc.UnaryInterceptor(observerRegistry.UnaryInterceptor),
+	}
+	s := grpc.NewServer(opts...)
+
+	// Register Services to the server.
+	pb.RegisterEchoServer(s, services.NewEchoServer())
+	identityServer := services.NewIdentityServer()
+	pb.RegisterIdentityServer(s, identityServer)
+	messagingServer := services.NewMessagingServer(identityServer)
+	pb.RegisterMessagingServer(s, messagingServer)
+	operationsServer := services.NewOperationsServer(messagingServer)
+	pb.RegisterTestingServer(s, services.NewTestingServer(observerRegistry))
+	lropb.RegisterOperationsServer(s, operationsServer)
+
+	var fb *fallback.FallbackServer
+	if len(fallbackPort) > 0 {
+		fb = fallback.NewServer(fallbackPort, "localhost"+port)
+		fb.StartBackground()
+	}
+
+	// Register reflection service on gRPC server.
+	reflection.Register(s)
+
+	var wait sync.WaitGroup
+	wait.Add(1)
+	go func() {
+		s.Serve(lis)
+		wait.Done()
+	}()
+	return &ShowcaseServers{s: s, fb: fb, wait: &wait}
+}
+
+// ShowcaseServers encapsulates information on running showcase servers, allowing for them to be
+// shutdown immediately or when they stop serving.
+type ShowcaseServers struct {
+	s    *grpc.Server
+	fb   *fallback.FallbackServer
+	wait *sync.WaitGroup
+}
+
+// Shutdown will immediately start a graceful shutdown of the servers.
+func (srv *ShowcaseServers) Shutdown() {
+	if srv.fb != nil {
+		srv.fb.Shutdown() // unfortunately, this always results in an en error log
+	}
+	if srv.s != nil {
+		srv.s.GracefulStop()
+	}
+}
+
+// Wait will wait until the servers stop serving and then call Shutdown() to shut them down
+// gracefully.
+func (srv *ShowcaseServers) Wait() {
+	if srv.wait != nil {
+		srv.wait.Wait()
+	}
+	srv.Shutdown()
+}

From d34b2aa88af1af02a7a40092bbe5adc5c97f645e Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Fri, 11 Oct 2019 17:47:50 -0700
Subject: [PATCH 29/39] Check for python plugin in prepare-to-qualify

---
 cmd/gapic-showcase/qualifier/prepare-to-qualify | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/prepare-to-qualify b/cmd/gapic-showcase/qualifier/prepare-to-qualify
index 9ea0fe366..a5133e8f5 100644
--- a/cmd/gapic-showcase/qualifier/prepare-to-qualify
+++ b/cmd/gapic-showcase/qualifier/prepare-to-qualify
@@ -2,19 +2,17 @@
 VENV_NAME=venv/qualify
 DATE=$(date +'%Y-%m-%d.%H-%M')
 
-BUILD_DIR=$(pwd)
-go build ../../gapic-showcase/
-
 verify() {
   name=$1
   msg=$2
-  $name --version  >& /dev/null || { echo "Please install $name to continue. $2" ; return 2 ; }  
+  which $name  || { echo "Please install $name to continue. $2" ; return 2 ; }  
 }
 
 verify python3 || return $?
 verify protoc "See https://github.com/protocolbuffers/protobuf/releases" || return $?
 verify pip || return $?
 verify curl || return $?
+verify protoc-gen-python_gapic || return $?
 
 export GAPIC_QUALIFY_DIR=${1:-$(mktemp -d -t qualify.${DATE}.XXXXXX)}
 pushd ${GAPIC_QUALIFY_DIR} >& /dev/null
@@ -29,7 +27,6 @@ echo '=== creating venv'  && \
   echo "=== installing pip" && \
   pip install sample-tester && \
   echo "=== installing showcase" && \
-  mv ${BUILD_DIR}/gapic-showcase . && \
   export PATH=$(pwd):${PATH}
 
 

From 44447c3704a2bf3c2f0e5bd408e81e8abff34496 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Sat, 12 Oct 2019 09:03:50 -0700
Subject: [PATCH 30/39] Make Packr part of the release process.

We now officially bundle the acceptance files within the Go binary.
---
 .gitignore               |  2 ++
 go.mod                   |  6 ++++++
 go.sum                   | 22 ++++++++++++++++++++++
 util/cmd/release/main.go | 19 ++++++++++++++-----
 util/compile_protos.go   |  2 +-
 util/execute.go          |  2 +-
 6 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index eee4026a7..21a203f78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 # Temp directories
 tmp/
 dist/
+**/packrd/*
+**/*-packr.go
diff --git a/go.mod b/go.mod
index c11e1338f..ffad3eb6b 100644
--- a/go.mod
+++ b/go.mod
@@ -6,8 +6,14 @@ require (
 	github.com/google/go-trace v0.0.0-20180203010202-d282251f7009
 	github.com/googleapis/gax-go/v2 v2.0.5
 	github.com/googleapis/grpc-fallback-go v0.1.3
+	github.com/hashicorp/go-version v1.2.0 // indirect
+	github.com/karrick/godirwalk v1.12.0 // indirect
+	github.com/mitchellh/gox v1.0.1 // indirect
+	github.com/rogpeppe/go-internal v1.5.0 // indirect
 	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.4.0
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.11.0
 	google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03
diff --git a/go.sum b/go.sum
index 5aaadfd82..efcc1d066 100644
--- a/go.sum
+++ b/go.sum
@@ -53,8 +53,11 @@ github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuT
 github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs=
 github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4=
 github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q=
+github.com/gobuffalo/packr v1.30.1 h1:hu1fuVR3fXEZR7rXNW3h8rqSML8EVAf6KNm0NKO/wKg=
 github.com/gobuffalo/packr/v2 v2.7.0 h1:snQei95nVOAtgY8rMiccTIPPYUXkuQws34FM/9/+UVg=
 github.com/gobuffalo/packr/v2 v2.7.0/go.mod h1:I+ICQFYFNPqTT7CUw7/RfMhluz683BO8FLH0wZgh/pQ=
+github.com/gobuffalo/packr/v2 v2.7.1 h1:n3CIW5T17T8v4GGK5sWXLVWJhCz7b5aNLSxW6gYim4o=
+github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -92,6 +95,10 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnnCPtE8=
+github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
@@ -107,9 +114,11 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/karrick/godirwalk v1.11.3 h1:ZrtYOzzHRzItdU1MvkK3CLlhC4m3YTWFgGyiBuSCQSY=
 github.com/karrick/godirwalk v1.11.3/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.12.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -121,6 +130,10 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI=
+github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4=
+github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -145,6 +158,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY=
 github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.5.0 h1:Usqs0/lDK/NqTkvrmKSwA/3XkZAs7ZAW/eLeQ2MVBTw=
+github.com/rogpeppe/go-internal v1.5.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -161,6 +176,8 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
@@ -190,6 +207,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
 golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -231,6 +250,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -248,6 +268,8 @@ golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/util/cmd/release/main.go b/util/cmd/release/main.go
index 84a659469..0c7010494 100644
--- a/util/cmd/release/main.go
+++ b/util/cmd/release/main.go
@@ -24,7 +24,7 @@ import (
 	"github.com/googleapis/gapic-showcase/util"
 )
 
-// This script is ran in CI when a new version tag is pushed to master. This script
+// This script is run in CI when a new version tag is pushed to master. This script
 // places the compiled proto descriptor set, a tarball of showcase-protos alongside its
 // dependencies, and the compiled executables of the gapic-showcase cli tool inside the
 // directory "dist"
@@ -103,10 +103,15 @@ func main() {
 	}
 	util.Execute(append(command, files...)...)
 
-	// Get cross compiler
-	// Mousetrap is a windows dependency that is not implicitly got since
-	// we only get the linux dependencies.
-	util.Execute("go", "get", "github.com/mitchellh/gox", "github.com/inconshreveable/mousetrap")
+	// Get Packr to generate the boxes with local files before compiling. The caller needs to
+	// make sure the installation directory is in the $PATH.
+	util.Execute("go", "get", "github.com/gobuffalo/packr/v2/packr2")
+	util.Execute("packr2")
+
+	// Get cross compiler. Mousetrap is a windows dependency that is not implicitly got since we
+	// only get the linux dependencies.  The caller needs to make sure the installation
+	// directory is in the $PATH.
+	util.Execute("go", "get", "-u", "github.com/mitchellh/gox", "github.com/inconshreveable/mousetrap")
 
 	// Compile binaries
 	stagingDir := filepath.Join("tmp", "binaries")
@@ -138,4 +143,8 @@ func main() {
 			filepath.Dir(files[0]),
 			filepath.Base(files[0]))
 	}
+
+	// Remove the generated Packr files from the source tree to avoid confusion.
+	util.Execute("packr2", "clean")
+
 }
diff --git a/util/compile_protos.go b/util/compile_protos.go
index 83b43f47b..992eaf428 100644
--- a/util/compile_protos.go
+++ b/util/compile_protos.go
@@ -25,7 +25,7 @@ import (
 
 // CompileProtos regenerates all of the generated source code for the Showcase
 // API including the generated messages, gRPC services, go gapic clients,
-// and the generated CLI. This must be ran from the root directory
+// and the generated CLI. This must be run from the root directory
 // of the gapic-showcase repository.
 func CompileProtos(version string) {
 	// Check if protoc is installed.
diff --git a/util/execute.go b/util/execute.go
index bbf660925..fcf076803 100644
--- a/util/execute.go
+++ b/util/execute.go
@@ -22,6 +22,6 @@ import (
 // Execute runs the given strings as a command.
 func Execute(args ...string) {
 	if output, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil {
-		log.Fatalf("%s", output)
+		log.Fatalf("%s running %s", output, args)
 	}
 }

From 34cb76bcd1aeaa38fe663325b840e04a00c44064 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 13:04:26 -0700
Subject: [PATCH 31/39] Remove startShowcase() and endProcess()

---
 cmd/gapic-showcase/qualifier/qualify.go | 32 +------------------------
 cmd/gapic-showcase/qualify.go           |  3 ++-
 2 files changed, 3 insertions(+), 32 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index 9fa3059ac..bc5427f8a 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -27,7 +27,7 @@ type Settings struct {
 	Verbose      bool
 }
 
-const showcaseCmd = "gapic-showcase" // TODO: Consider running in-process
+const showcaseCmd = "gapic-showcase"
 
 func Run(settings *Settings) {
 	// TODO: Return an error rather than exiting. We can return an error type ErrorCode that
@@ -109,36 +109,6 @@ func GetTestScenarios(settings *Settings) ([]*Scenario, error) {
 	return allScenarios, nil
 }
 
-// startShowcase starts the Showcase server and returns its PID
-func startShowcase() (*exec.Cmd, error) {
-	// TODO: fill in
-	trace.Trace("")
-	cmd := exec.Command(showcaseCmd)
-	err := cmd.Start()
-	return cmd, err
-}
-
-// endProcess ends the process with the specified PID
-func endProcess(cmd *exec.Cmd) error {
-	// TODO: fill in
-	trace.Trace("cmd=%v)", cmd)
-	process := cmd.Process
-	if err := process.Signal(os.Interrupt); err != nil {
-		msg := fmt.Sprintf("could not end process %v normally", process.Pid)
-		if err := process.Kill(); err != nil {
-			msg = fmt.Sprintf("%s; killing failed. Please kill manually!", msg)
-			log.Printf(msg)
-			return fmt.Errorf(msg)
-		}
-		msg = fmt.Sprintf("%s but killing it succeeded", msg)
-		log.Printf(msg)
-	}
-	trace.Trace("waiting for process to end: %v (%v)", process.Pid, cmd.Path)
-	cmd.Wait()
-	trace.Trace("process ended:              %v (%v) ended", process.Pid, cmd.Path)
-	return nil
-}
-
 // getAllFiles returns a list of all the files (excluding directories)
 // at any level under `root`.
 func getAllFiles(root string) ([]string, error) {
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index c7c784f65..879ff938b 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -28,10 +28,11 @@ func init() {
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			servers := RunShowcase(strconv.Itoa(settings.ShowcasePort), "")
+			defer servers.Shutdown()
+
 			settings.Language = args[0]
 			trace.Trace("settings: %v", settings)
 			qualifier.Run(settings)
-			servers.Shutdown()
 		},
 	}
 	rootCmd.AddCommand(qualifyCmd)

From 9265b4146772617ce0241784574606051646133b Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 13:05:08 -0700
Subject: [PATCH 32/39] Add sanity check in case the assets are empty

Otherwise we'll trivially succeed on zero checks.
---
 cmd/gapic-showcase/qualifier/assets.go  | 11 +++++++----
 cmd/gapic-showcase/qualifier/qualify.go |  4 +++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/assets.go b/cmd/gapic-showcase/qualifier/assets.go
index f3d52f4bb..521050716 100644
--- a/cmd/gapic-showcase/qualifier/assets.go
+++ b/cmd/gapic-showcase/qualifier/assets.go
@@ -15,15 +15,18 @@ var SchemaSuite *packr.Box
 const fileTypeSampleConfig = "com.google.api.codegen.SampleConfigProto"
 const fileTypeProtobuf = "proto"
 
-func GetAssets() {
-	// I believe we can't pass the arguments into a function
-	// because otherwise packr won't be able to recognize these
-	// paths should be packed.
+func GetAssets() error {
 	AcceptanceSuite = packr.New("acceptance suite", "acceptance_suite")
 	SchemaSuite = packr.New("schema", "../../../schema")
 
+	if len(AcceptanceSuite.List()) == 0 || len(SchemaSuite.List()) == 0 {
+		return fmt.Errorf("release error: some of the asset boxes are empty")
+	}
+
 	traceBox(AcceptanceSuite)
 	traceBox(SchemaSuite)
+
+	return nil
 }
 
 func traceBox(box *packr.Box) {
diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index bc5427f8a..7204ec553 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -41,7 +41,9 @@ func Run(settings *Settings) {
 		RetScenarioFailure
 	)
 
-	GetAssets()
+	if err := GetAssets(); err != nil {
+		os.Exit(RetCodeInternalError)
+	}
 
 	if err := checkDependencies(); err != nil {
 		os.Exit(RetCodeFailedDependencies)

From 36a69f8c4eec4a09a8720396dd10c61649f03f0f Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 13:13:28 -0700
Subject: [PATCH 33/39] Remove getAllFiles

---
 cmd/gapic-showcase/qualifier/qualify.go | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index 7204ec553..af8d85284 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -110,22 +110,3 @@ func GetTestScenarios(settings *Settings) ([]*Scenario, error) {
 	}
 	return allScenarios, nil
 }
-
-// getAllFiles returns a list of all the files (excluding directories)
-// at any level under `root`.
-func getAllFiles(root string) ([]string, error) {
-	trace.Trace("root=%q", root)
-	allFiles := []string{}
-	err := filepath.Walk(root,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-			allFiles = append(allFiles, path)
-			return nil
-		})
-	return allFiles, err
-}

From 25eb7477dd0f08915b5a216a627a9b5ed72faae2 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 16:03:20 -0700
Subject: [PATCH 34/39] Document code and add licenses

---
 cmd/gapic-showcase/qualifier/assets.go        | 112 ------------
 cmd/gapic-showcase/qualifier/dependencies.go  | 160 ++++++++++++++++++
 cmd/gapic-showcase/qualifier/generator.go     |  40 +++--
 .../qualifier/prepare-to-qualify              |  24 ++-
 cmd/gapic-showcase/qualifier/qualify.go       |  71 +++-----
 cmd/gapic-showcase/qualifier/scenario.go      | 135 ++++++++++-----
 cmd/gapic-showcase/qualify.go                 |  26 ++-
 7 files changed, 353 insertions(+), 215 deletions(-)
 delete mode 100644 cmd/gapic-showcase/qualifier/assets.go
 create mode 100644 cmd/gapic-showcase/qualifier/dependencies.go

diff --git a/cmd/gapic-showcase/qualifier/assets.go b/cmd/gapic-showcase/qualifier/assets.go
deleted file mode 100644
index 521050716..000000000
--- a/cmd/gapic-showcase/qualifier/assets.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package qualifier
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	packr "github.com/gobuffalo/packr/v2"
-	trace "github.com/google/go-trace"
-)
-
-var AcceptanceSuite *packr.Box
-var SchemaSuite *packr.Box
-
-const fileTypeSampleConfig = "com.google.api.codegen.SampleConfigProto"
-const fileTypeProtobuf = "proto"
-
-func GetAssets() error {
-	AcceptanceSuite = packr.New("acceptance suite", "acceptance_suite")
-	SchemaSuite = packr.New("schema", "../../../schema")
-
-	if len(AcceptanceSuite.List()) == 0 || len(SchemaSuite.List()) == 0 {
-		return fmt.Errorf("release error: some of the asset boxes are empty")
-	}
-
-	traceBox(AcceptanceSuite)
-	traceBox(SchemaSuite)
-
-	return nil
-}
-
-func traceBox(box *packr.Box) {
-	trace.Trace("suite %q has %d entries", box.Name, len(box.List()))
-}
-
-type FilesByDir struct {
-	Directory string
-	Files     []string // filenames are relative paths from `Directory`
-}
-
-func GetFilesByDir(box *packr.Box) []*FilesByDir {
-	filesInAllDirs := []*FilesByDir{}
-	var filesInThisDir *FilesByDir
-	commitThisDir := func() {
-		if filesInThisDir != nil {
-			filesInAllDirs = append(filesInAllDirs, filesInThisDir)
-			filesInThisDir = nil
-		}
-	}
-
-	// We assume allFiles is returned in a top-down order, so that
-	// all files under each first level directory appear
-	// contiguously.
-	allFiles := box.List()
-	previousDir := ""
-	for _, oneFile := range allFiles {
-		dir := strings.Split(oneFile, string(os.PathSeparator))[0]
-		if dir != previousDir {
-			commitThisDir()
-			filesInThisDir = &FilesByDir{Directory: dir}
-		}
-		previousDir = dir
-		filesInThisDir.Files = append(filesInThisDir.Files, oneFile[len(dir)+len(string(os.PathSeparator)):])
-	}
-	commitThisDir()
-
-	return filesInAllDirs
-}
-
-// GetMatchingFiles returns the a list of files in `box` matching
-// `srcPath`. `srcPath` should end with `os.PathSeparator` iff it refers
-// to a directory.
-//
-// As a convenience, this also returns an additional value, useful
-// for copying files within matched directories:
-//
-//  * `replacePath`: a copy of `dstPath` (which is assumed to have no
-//     trailing separator), with `os.PathSeparator` appended if
-//     `srcPath` refers to a directory
-func GetMatchingFiles(box *packr.Box, dstPath string, srcPath string) (files []string, replacePath string, err error) {
-
-	trace.Trace("reading %q", srcPath)
-
-	replacePath = dstPath
-
-	// If `srcPath` specifies a single file, match just that.
-	if !strings.HasSuffix(srcPath, string(os.PathSeparator)) {
-		if !box.Has(srcPath) {
-			err = fmt.Errorf("file box %q has no file %q", box.Name, srcPath)
-			return
-		}
-		files = []string{srcPath}
-		return
-	}
-
-	// Let the caller replace the directory part of the path with
-	// the separator included.
-	replacePath = dstPath + string(os.PathSeparator)
-
-	for _, entry := range box.List() {
-		if strings.HasPrefix(entry, srcPath) {
-			files = append(files, entry)
-		}
-	}
-	if len(files) == 0 {
-		err = fmt.Errorf("file box %q has no files matching %q", box.Name, srcPath)
-		return
-	}
-
-	trace.Trace("obtained %d files", len(files))
-	return
-}
diff --git a/cmd/gapic-showcase/qualifier/dependencies.go b/cmd/gapic-showcase/qualifier/dependencies.go
new file mode 100644
index 000000000..ca1b28691
--- /dev/null
+++ b/cmd/gapic-showcase/qualifier/dependencies.go
@@ -0,0 +1,160 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qualifier
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	packr "github.com/gobuffalo/packr/v2"
+	trace "github.com/google/go-trace"
+)
+
+var (
+	acceptanceSuite *packr.Box // check suite and their files: protos, configs, tests
+	schemaSuite     *packr.Box // the schema files that may be needed by various suite checks
+)
+
+const (
+	// various file types for classifying suite files
+	fileTypeSampleConfig = "com.google.api.codegen.SampleConfigProto"
+	fileTypeProtobuf     = "proto"
+
+	// external command to run
+	cmdSampleTester = "sample-tester"
+)
+
+// getAssets loads the assets needed by the qualifier suite. These are Packr boxes with the files
+// obtained from the source tree.
+func getAssets() error {
+	acceptanceSuite = packr.New("acceptance suite", "acceptance_suite")
+	schemaSuite = packr.New("schema", "../../../schema")
+
+	if len(acceptanceSuite.List()) == 0 || len(schemaSuite.List()) == 0 {
+		return fmt.Errorf("release error: some of the asset boxes are empty")
+	}
+
+	traceBox(acceptanceSuite)
+	traceBox(schemaSuite)
+	return nil
+}
+
+// checkDependencies checks that any external dependencies (typically programs that the qualifier
+// suite calls) are accessible in this invocation.
+func checkDependencies() error {
+	notFound := []string{}
+	trace.Trace("")
+
+	sampleTesterPath, err := exec.LookPath(cmdSampleTester)
+	if err != nil {
+		notFound = append(notFound, cmdSampleTester)
+	}
+
+	if len(notFound) > 0 {
+		msg := fmt.Sprintf("could not find dependencies in $PATH: %q", notFound)
+		log.Printf(msg)
+		return fmt.Errorf(msg)
+	}
+	trace.Trace("found %q: %s", cmdSampleTester, sampleTesterPath)
+	return nil
+}
+
+func traceBox(box *packr.Box) {
+	trace.Trace("suite %q has %d entries", box.Name, len(box.List()))
+}
+
+// filesByDir contains a list of all the files under the given directory. Each entry in `Files`
+// includes the path components for `Directory`.
+type filesByDir struct {
+	directory string
+	files     []string // filenames are relative paths from `Directory`
+}
+
+// getFilesByDir returns a list of `filesByDir` objects, one per top-level directory in `box`.
+func getFilesByDir(box *packr.Box) []*filesByDir {
+	filesInAllDirs := []*filesByDir{}
+	var filesInThisDir *filesByDir
+	commitThisDir := func() {
+		if filesInThisDir != nil {
+			filesInAllDirs = append(filesInAllDirs, filesInThisDir)
+			filesInThisDir = nil
+		}
+	}
+
+	// We assume allFiles is returned in a top-down order, so that
+	// all files under each first level directory appear
+	// contiguously.
+	allFiles := box.List()
+	previousDir := ""
+	for _, oneFile := range allFiles {
+		dir := strings.Split(oneFile, string(os.PathSeparator))[0]
+		if dir != previousDir {
+			commitThisDir()
+			filesInThisDir = &filesByDir{directory: dir}
+		}
+		previousDir = dir
+		filesInThisDir.files = append(filesInThisDir.files, oneFile[len(dir)+len(string(os.PathSeparator)):])
+	}
+	commitThisDir()
+
+	return filesInAllDirs
+}
+
+// GetMatchingFiles returns the a list of files in `box` matching
+// `srcPath`. `srcPath` should end with `os.PathSeparator` iff it refers
+// to a directory.
+//
+// As a convenience, this also returns an additional value, useful
+// for copying files within matched directories:
+//
+//  * `replacePath`: a copy of `dstPath` (which is assumed to have no
+//     trailing separator), with `os.PathSeparator` appended if
+//     `srcPath` refers to a directory
+func GetMatchingFiles(box *packr.Box, dstPath string, srcPath string) (files []string, replacePath string, err error) {
+
+	trace.Trace("reading %q", srcPath)
+
+	replacePath = dstPath
+
+	// If `srcPath` specifies a single file, match just that.
+	if !strings.HasSuffix(srcPath, string(os.PathSeparator)) {
+		if !box.Has(srcPath) {
+			err = fmt.Errorf("file box %q has no file %q", box.Name, srcPath)
+			return
+		}
+		files = []string{srcPath}
+		return
+	}
+
+	// Let the caller replace the directory part of the path with
+	// the separator included.
+	replacePath = dstPath + string(os.PathSeparator)
+
+	for _, entry := range box.List() {
+		if strings.HasPrefix(entry, srcPath) {
+			files = append(files, entry)
+		}
+	}
+	if len(files) == 0 {
+		err = fmt.Errorf("file box %q has no files matching %q", box.Name, srcPath)
+		return
+	}
+
+	trace.Trace("obtained %d files", len(files))
+	return
+}
diff --git a/cmd/gapic-showcase/qualifier/generator.go b/cmd/gapic-showcase/qualifier/generator.go
index 69da1a4f4..3faa7f008 100644
--- a/cmd/gapic-showcase/qualifier/generator.go
+++ b/cmd/gapic-showcase/qualifier/generator.go
@@ -1,3 +1,17 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package qualifier
 
 import (
@@ -15,15 +29,17 @@ import (
 // will be run as a protoc plugin unless `isMonolith` is set, in which case gapic-generator will be
 // invoked directly.
 type Generator struct {
-	Language        string
-	PluginDirectory string
-	PluginOptions   string
-	isMonolith      bool
+	Language   string // the language in which to generate GAPICs and samples
+	Directory  string // the directory in which to find the generator
+	Options    string // any options
+	isMonolith bool   // TODO: implement this functionality and export this field
 }
 
-// Run will run the generator from `work_dir`, obtaining the required files from filesByType.
+// Run runs the generator `gen` from `work_dir`, creating it if necessary and obtaining the required
+// files from `filesByType`. The generated output is placed in a `generated/` sub-directory.
 func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]byte, *os.ProcessState, error) {
-	generationDir := "generated"
+	const generationDir = "generated"
+
 	if gen.isMonolith {
 		return nil, nil, fmt.Errorf("monolith not implemented yet")
 	}
@@ -32,10 +48,12 @@ func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]by
 		return nil, nil, err
 	}
 
+	// Construct the various arguments to invoke the generator as a protoc plugin.
+
 	optionFlag := fmt.Sprintf("--%s_gapic_opt", gen.Language)
 	allOptions := []string{}
-	if len(gen.PluginOptions) > 0 {
-		allOptions = append(allOptions, optionFlag, gen.PluginOptions)
+	if len(gen.Options) > 0 {
+		allOptions = append(allOptions, optionFlag, gen.Options)
 	}
 
 	sampleConfigFiles, _ := filesByType[fileTypeSampleConfig]
@@ -49,12 +67,14 @@ func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]by
 		fmt.Sprintf("--%s_gapic_out", gen.Language), fmt.Sprintf("./%s", generationDir),
 	}
 	cmdParts = append(cmdParts, allOptions...)
-	if len(gen.PluginDirectory) > 0 {
-		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gen.PluginDirectory, gen.Language))
+	if len(gen.Directory) > 0 {
+		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gen.Directory, gen.Language))
 
 	}
 	cmdParts = append(cmdParts, filesByType[fileTypeProtobuf]...)
 
+	// Execute the command, clear all but internal errors, return.
+
 	cmdString := strings.Join(cmdParts, " ")
 	trace.Trace("running: %s", cmdString)
 
diff --git a/cmd/gapic-showcase/qualifier/prepare-to-qualify b/cmd/gapic-showcase/qualifier/prepare-to-qualify
index a5133e8f5..197d52c11 100644
--- a/cmd/gapic-showcase/qualifier/prepare-to-qualify
+++ b/cmd/gapic-showcase/qualifier/prepare-to-qualify
@@ -1,7 +1,27 @@
 #!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is a tool for developers to invoke to prepare for local
+# development. It sets up the environment needed to run
+# `gapic-showcase-qualify`.
+
 VENV_NAME=venv/qualify
 DATE=$(date +'%Y-%m-%d.%H-%M')
 
+# Check external dependencies
+
 verify() {
   name=$1
   msg=$2
@@ -14,10 +34,12 @@ verify pip || return $?
 verify curl || return $?
 verify protoc-gen-python_gapic || return $?
 
+
+# Set up environment
+
 export GAPIC_QUALIFY_DIR=${1:-$(mktemp -d -t qualify.${DATE}.XXXXXX)}
 pushd ${GAPIC_QUALIFY_DIR} >& /dev/null
 
-
 VENV_CREATE="python3 -m venv $VENV_NAME"
 
 echo '=== creating venv'  && \
diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index af8d85284..60e92bb55 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -1,34 +1,35 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package qualifier
 
 import (
 	"fmt"
-	"log"
 	"os"
-	"os/exec"
-	"path/filepath"
 
 	trace "github.com/google/go-trace"
 )
 
-/* Development instructions
-
-. ./prepare-to-qualify
-
-=== Sample run ======
-go run *.go
-
-
-*/
-
+// Settings contains the settings for a run of the qualification suite.
 type Settings struct {
 	Generator
 	ShowcasePort int
-	Timestamp    string
+	Timestamp    string // for tracing and diagnostics
 	Verbose      bool
 }
 
-const showcaseCmd = "gapic-showcase"
-
+// Run runs the qualification suite using the values in `settings`.
 func Run(settings *Settings) {
 	// TODO: Return an error rather than exiting. We can return an error type ErrorCode that
 	// wraps the current errors and includes the appropriate return code, and change main() so
@@ -41,7 +42,7 @@ func Run(settings *Settings) {
 		RetScenarioFailure
 	)
 
-	if err := GetAssets(); err != nil {
+	if err := getAssets(); err != nil {
 		os.Exit(RetCodeInternalError)
 	}
 
@@ -49,7 +50,7 @@ func Run(settings *Settings) {
 		os.Exit(RetCodeFailedDependencies)
 	}
 
-	allScenarios, err := GetTestScenarios(settings)
+	allScenarios, err := getTestScenarios(settings)
 	if err != nil {
 		os.Exit(RetCodeInternalError)
 	}
@@ -71,39 +72,19 @@ func Run(settings *Settings) {
 	}
 }
 
-func checkDependencies() error {
-	sampleTesterCmd := "sample-tester"
-	notFound := []string{}
-	trace.Trace("")
-
-	sampleTesterPath, err := exec.LookPath(sampleTesterCmd)
-	if err != nil {
-		notFound = append(notFound, sampleTesterCmd)
-	}
-
-	if len(notFound) > 0 {
-		msg := fmt.Sprintf("could not find dependencies in $PATH: %q", notFound)
-		log.Printf(msg)
-		return fmt.Errorf(msg)
-	}
-	trace.Trace("found %q: %s", sampleTesterCmd, sampleTesterPath)
-	return nil
-}
-
-// GetTestScenarios returns a list of Scenario as found in the specified
-// scenario root directory.
-func GetTestScenarios(settings *Settings) ([]*Scenario, error) {
+// getTestScenarios returns a list of Scenario as found in the acceptanceSuite.
+func getTestScenarios(settings *Settings) ([]*Scenario, error) {
 	allScenarios := []*Scenario{}
-	allScenarioConfigs := GetFilesByDir(AcceptanceSuite)
+	allScenarioConfigs := getFilesByDir(acceptanceSuite)
 	for _, config := range allScenarioConfigs {
 		newScenario := &Scenario{
-			name:         config.Directory,
+			name:         config.directory,
 			timestamp:    settings.Timestamp,
 			showcasePort: settings.ShowcasePort,
 			generator:    &settings.Generator,
-			files:        config.Files,
-			fileBox:      AcceptanceSuite,
-			schemaBox:    SchemaSuite,
+			files:        config.files,
+			fileBox:      acceptanceSuite,
+			schemaBox:    schemaSuite,
 		}
 		trace.Trace("adding scenario %#v", newScenario)
 		allScenarios = append(allScenarios, newScenario)
diff --git a/cmd/gapic-showcase/qualifier/scenario.go b/cmd/gapic-showcase/qualifier/scenario.go
index 1ce920b6e..833534402 100644
--- a/cmd/gapic-showcase/qualifier/scenario.go
+++ b/cmd/gapic-showcase/qualifier/scenario.go
@@ -1,3 +1,17 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package qualifier
 
 import (
@@ -14,55 +28,58 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-const includeFilePrefix = "include."
-const includeFileDirectorySeparator = '/'
+// Constants pertaining to the syntax of include files. Include files are akin to symbolic links:
+// they reference content that will be copied at run-time from the indicated location to the include
+// file's location under the indicated names. For example, an include file called "include.boren"
+// with contents "api-common-protos/google/" will be replaced at run-time with a directory called
+// "boren" whose contents will be a copy of everything under "api-common-protos/google/"
+//
+// We need some mechanism for including common protos because the protos under test may reference
+// them. This particular scheme is also useful in that it allows us to explicitly pass to the
+// generator only the protos explicitly listed in the acceptance check, rather than also having to
+// include all the protos that are transitively included.
+const (
+	// In a qualifier suite directory, any file that begins with this prefix is considered an
+	// include file. The rest of the filename after the prefix is the destination filename to
+	// which the files referenced by the content of the include file will be copied.
+	includeFilePrefix = "include."
+
+	// In the contents of an include file, this symbol will be treated as the directory
+	// separator. At run time, it will be replaced the appropriate os.PathSeparator.
+	includeFileDirectorySeparator = '/'
+)
 
 type Scenario struct {
 	// Every element in `files` corresponds to a file in `fileBox` under
 	// a directory called `name`, which is NOT part of the string
 	// element itself.
-	name        string
-	timestamp   string
-	files       []string
-	fileBox     *packr.Box
-	schemaBox   *packr.Box
-	sandbox     string
-	filesByType map[string][]string
+	name        string              // name of the scenario
+	timestamp   string              // timestamp for debug and trace messages
+	files       []string            // a flat list of files in the scenario
+	fileBox     *packr.Box          // the files explicitly specified in the scenario
+	schemaBox   *packr.Box          // additional files that may be referenced by include files in a scenario
+	sandbox     string              // the sandbox created to run this scenario
+	filesByType map[string][]string // scenario files partitioned by file type
 
 	generator    *Generator
-	showcasePort int
+	showcasePort int // the port of the running showcase server against which samples will run
 
-	generationOutput       []byte
-	generationProcessState *os.ProcessState
-	generationPassed       bool
-	sampleTestsPassed      bool
+	generationOutput       []byte           // combined output of the generation process
+	generationProcessState *os.ProcessState // info on the completed generation process
+	generationPassed       bool             // whether all generation checks passed
+	sampleTestsPassed      bool             // whether all sample tests specified passed
 }
 
-func (scenario *Scenario) sandboxPath(relativePath string) string {
-	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
-		panic("scenario name not set yet when trying to get sandbox filename")
-	}
-	if len(scenario.sandbox) == 0 {
-		trace.Trace("sandbox name not set yet: relativePath==%q", relativePath)
-		panic("sandbox name not set yet when trying to get sandbox filename")
-	}
-
-	return filepath.Join(scenario.sandbox, relativePath)
-}
-
-func (scenario *Scenario) fileBoxPath(relativePath string) string {
-	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
-		panic("scenario name not set yet when trying to get box filename")
-	}
-	return filepath.Join(scenario.name, relativePath)
-}
-
-func (scenario *Scenario) fromFileBox(relativePath string) ([]byte, error) {
-	return scenario.fileBox.Find(scenario.fileBoxPath(relativePath))
+// Success returns true iff all the expectations for this scenario were met.
+func (scenario *Scenario) Success() bool {
+	return scenario.generationPassed && // the generation checks passed and...
+		(!scenario.generationProcessState.Success() || // ...either generation failed, or it succeeded with ...
+			(len(scenario.filesByType["test/samples"]) == 0 || scenario.sampleTestsPassed)) // all sample tests, if any, passing
 }
 
+// Run runs this particular acceptance scenario by creating a sandbox in which it generates
+// libraries and samples and then tests that the generation and any sample tests performed as
+// expected.
 func (scenario *Scenario) Run() error {
 	if err := scenario.classifyConfigs(); err != nil {
 		return fmt.Errorf("could not classify config files for scenario %q: %w", scenario.name, err)
@@ -81,6 +98,7 @@ func (scenario *Scenario) Run() error {
 	return nil
 }
 
+// Generate generates the client libraries and samples for this scenario.
 func (scenario *Scenario) Generate() error {
 	var err error
 	scenario.generationOutput, scenario.generationProcessState, err = scenario.generator.Run(scenario.sandbox, scenario.filesByType)
@@ -89,21 +107,47 @@ func (scenario *Scenario) Generate() error {
 	return err
 }
 
+// CheckGeneration verifies that generation matched configured expectations when it succeeded or
+// failed.
 func (scenario *Scenario) CheckGeneration() {
 	// TODO: Fill in in order to handle expected, configured failures
 	trace.Trace("CheckGeneration")
 	scenario.generationPassed = scenario.generationProcessState.Success()
 }
 
+// RunTests runs tests on the generated samples.
 func (scenario *Scenario) RunTests() {
 	// TODO: Fill in
 	trace.Trace("RunTests (TODO)")
 }
 
-func (scenario *Scenario) Success() bool {
-	return scenario.generationPassed && // the generation checks passed and...
-		(!scenario.generationProcessState.Success() || // ...either generation failed, or it succeeded with ...
-			(len(scenario.filesByType["test/samples"]) == 0 || scenario.sampleTestsPassed)) // all sample tests, if any, passing
+// sandboxPath returns `relativePath` re-rooted at `scenario.sandbox`.
+func (scenario *Scenario) sandboxPath(relativePath string) string {
+	if len(scenario.name) == 0 {
+		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
+		panic("scenario name not set yet when trying to get sandbox filename")
+	}
+	if len(scenario.sandbox) == 0 {
+		trace.Trace("sandbox name not set yet: relativePath==%q", relativePath)
+		panic("sandbox name not set yet when trying to get sandbox filename")
+	}
+
+	return filepath.Join(scenario.sandbox, relativePath)
+}
+
+// fileBoxPath returns relativePath re-rooted at scenario.name, so that it references an object in
+// `scenario.fileBox`.
+func (scenario *Scenario) fileBoxPath(relativePath string) string {
+	if len(scenario.name) == 0 {
+		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
+		panic("scenario name not set yet when trying to get box filename")
+	}
+	return filepath.Join(scenario.name, relativePath)
+}
+
+// fromFileBox returns the file named by `relativePath` from `scenario.fileBox`.
+func (scenario *Scenario) fromFileBox(relativePath string) ([]byte, error) {
+	return scenario.fileBox.Find(scenario.fileBoxPath(relativePath))
 }
 
 // classifyConfigs classifies all the files in the scenario, storing the results in
@@ -168,6 +212,8 @@ func (scenario *Scenario) getFileTypes(thisFile string) (fileTypes []string, err
 	return fileTypes, err
 }
 
+// yamlDocTypes returns a list of the document types encountered in the (possibly multi-document)
+// YAML payload `fileContent`.
 func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 	type docSchema struct {
 		Type string
@@ -190,8 +236,9 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 	return docTypes, nil
 }
 
+// createSandbox creates a sandbox directory within which all the generation and checks for this
+// scenario will be run.
 func (scenario *Scenario) createSandbox() (err error) {
-
 	if scenario.sandbox, err = ioutil.TempDir("", fmt.Sprintf("showcase-qualify.%s.%s.", scenario.timestamp, scenario.name)); err != nil {
 		err := fmt.Errorf("could not create sandbox: %w", err)
 		trace.Trace(err)
@@ -260,10 +307,14 @@ func (scenario *Scenario) getIncludes(includes []string) error {
 	return nil
 }
 
+// copyFiles copies the files listed in `files` from `scenario.fromFileBox` to `scenario.sandbox`.
 func (scenario *Scenario) copyFiles(files []string) error {
 	return scenario.copyFilesReplacePath(files, scenario.fromFileBox, "", "")
 }
 
+// copyFilesReplacePaths copies the files listed in `files` from the source in `fromBox` to the
+// `scenario.sandbox`. In so doing, it replaces a leading `replaceSrc` prefix in each filename with
+// `replaceDst`.
 func (scenario *Scenario) copyFilesReplacePath(files []string, fromBox func(string) ([]byte, error), replaceSrc, replaceDst string) (err error) {
 	const filePermissions = 0555
 	replace := len(replaceSrc) > 0
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index 879ff938b..a80269098 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -1,3 +1,17 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (
@@ -11,8 +25,6 @@ import (
 )
 
 func init() {
-	trace.On(false) // set to true for debugging
-
 	var timestamp string
 	timestamp = time.Now().Format("20060102.150405")
 	trace.Trace("timestamp = %q", timestamp)
@@ -27,6 +39,10 @@ func init() {
 		Short: "Tests a provided GAPIC generator against an acceptance suite",
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			// TODO: Consider moving this to a more central place for debugging all of
+			// showcase.
+			trace.On(false) // set to true for debugging
+
 			servers := RunShowcase(strconv.Itoa(settings.ShowcasePort), "")
 			defer servers.Shutdown()
 
@@ -37,16 +53,16 @@ func init() {
 	}
 	rootCmd.AddCommand(qualifyCmd)
 	qualifyCmd.Flags().StringVarP(
-		&settings.PluginDirectory,
+		&settings.Directory,
 		"dir",
 		"d",
 		"",
 		"The directory in which to find the protoc plugin implementing the given GAPIC generator")
 	qualifyCmd.Flags().StringVarP(
-		&settings.PluginOptions,
+		&settings.Options,
 		"options",
 		"o",
 		"",
-		"The options to pass to the protoc plugin in order to generate a GAPIC for the showcase Echo service")
+		"The options to pass to the generator in order to generate a GAPIC for the showcase Echo service")
 
 }

From c80a2256b9ae3db0b1086059ac7806e6b52c8534 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 16:25:58 -0700
Subject: [PATCH 35/39] Update go.mod and go.sum after rebasing to latest
 main/HEAD

---
 go.mod | 1 +
 go.sum | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/go.mod b/go.mod
index ffad3eb6b..2c6db6eea 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,7 @@ module github.com/googleapis/gapic-showcase
 
 require (
 	cloud.google.com/go v0.47.0
+	github.com/gobuffalo/packr/v2 v2.7.1
 	github.com/golang/protobuf v1.3.2
 	github.com/google/go-trace v0.0.0-20180203010202-d282251f7009
 	github.com/googleapis/gax-go/v2 v2.0.5
diff --git a/go.sum b/go.sum
index efcc1d066..682161e87 100644
--- a/go.sum
+++ b/go.sum
@@ -293,6 +293,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010171213-8abd42400456 h1:LR16zMCx87X52rsLOtnByklL2K/xWUKAo1Nm7AA4HA0=
 golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=

From a7c1bf72e7ce2f1dc01bd09001a189dbc749612f Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 14 Oct 2019 16:36:11 -0700
Subject: [PATCH 36/39] Rename the demo test suite so it's called "basic-check"

---
 .../acceptance_suite/{passing => basic-check}/echo.proto          | 0
 .../{passing => basic-check}/foo/bar/include.wdREAD               | 0
 .../acceptance_suite/{passing => basic-check}/include.google      | 0
 .../acceptance_suite/{passing => basic-check}/include.vcREAD      | 0
 .../acceptance_suite/{passing => basic-check}/sample.yaml         | 0
 5 files changed, 0 insertions(+), 0 deletions(-)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/{passing => basic-check}/echo.proto (100%)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/{passing => basic-check}/foo/bar/include.wdREAD (100%)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/{passing => basic-check}/include.google (100%)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/{passing => basic-check}/include.vcREAD (100%)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/{passing => basic-check}/sample.yaml (100%)

diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/passing/echo.proto b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/echo.proto
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/passing/echo.proto
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/echo.proto
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/passing/foo/bar/include.wdREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.wdREAD
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/passing/foo/bar/include.wdREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.wdREAD
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.google b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.google
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.google
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.google
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.vcREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.vcREAD
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/passing/include.vcREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.vcREAD
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/passing/sample.yaml b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/sample.yaml
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/passing/sample.yaml
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/sample.yaml

From a41706aaa385bcac762f04e9af12976f2abf4ba8 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Tue, 15 Oct 2019 10:16:46 -0700
Subject: [PATCH 37/39] Minor cosmetic tweaks

---
 cmd/gapic-showcase/qualifier/dependencies.go    | 2 +-
 cmd/gapic-showcase/qualifier/prepare-to-qualify | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/dependencies.go b/cmd/gapic-showcase/qualifier/dependencies.go
index ca1b28691..64a62e076 100644
--- a/cmd/gapic-showcase/qualifier/dependencies.go
+++ b/cmd/gapic-showcase/qualifier/dependencies.go
@@ -82,7 +82,7 @@ func traceBox(box *packr.Box) {
 // includes the path components for `Directory`.
 type filesByDir struct {
 	directory string
-	files     []string // filenames are relative paths from `Directory`
+	files     []string // filenames are paths relative to `Directory`
 }
 
 // getFilesByDir returns a list of `filesByDir` objects, one per top-level directory in `box`.
diff --git a/cmd/gapic-showcase/qualifier/prepare-to-qualify b/cmd/gapic-showcase/qualifier/prepare-to-qualify
index 197d52c11..b2bb9fc54 100644
--- a/cmd/gapic-showcase/qualifier/prepare-to-qualify
+++ b/cmd/gapic-showcase/qualifier/prepare-to-qualify
@@ -15,7 +15,7 @@
 
 # This script is a tool for developers to invoke to prepare for local
 # development. It sets up the environment needed to run
-# `gapic-showcase-qualify`.
+# `gapic-showcase qualify`.
 
 VENV_NAME=venv/qualify
 DATE=$(date +'%Y-%m-%d.%H-%M')

From 46f760f77b885eaf24c069d25e3f81831a5ff8d6 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 21 Oct 2019 10:52:35 -0700
Subject: [PATCH 38/39] Code review changes: code

---
 cmd/gapic-showcase/qualifier/dependencies.go |  73 +++++-----
 cmd/gapic-showcase/qualifier/generator.go    |  19 ++-
 cmd/gapic-showcase/qualifier/qualify.go      |  41 +++---
 cmd/gapic-showcase/qualifier/scenario.go     | 144 ++++++++-----------
 cmd/gapic-showcase/qualify.go                |  12 +-
 5 files changed, 140 insertions(+), 149 deletions(-)

diff --git a/cmd/gapic-showcase/qualifier/dependencies.go b/cmd/gapic-showcase/qualifier/dependencies.go
index 64a62e076..87dd50e56 100644
--- a/cmd/gapic-showcase/qualifier/dependencies.go
+++ b/cmd/gapic-showcase/qualifier/dependencies.go
@@ -31,9 +31,17 @@ var (
 )
 
 const (
-	// various file types for classifying suite files
+	// various types that classify file contents, based on the file extension and/or YAML
+	// document types.
 	fileTypeSampleConfig = "com.google.api.codegen.SampleConfigProto"
+	fileTypeSampleTest   = "test/samples"
 	fileTypeProtobuf     = "proto"
+	fileTypeUnknown      = "(UNKNOWN)"
+
+	// each file in a scenario directory is either a scenario file that helps define a
+	// scenario, or an include file needed to support a scenario (eg commonly used types)
+	fileTypeScenario = "scenario"
+	fileTypeInclude  = "include"
 
 	// external command to run
 	cmdSampleTester = "sample-tester"
@@ -87,12 +95,12 @@ type filesByDir struct {
 
 // getFilesByDir returns a list of `filesByDir` objects, one per top-level directory in `box`.
 func getFilesByDir(box *packr.Box) []*filesByDir {
-	filesInAllDirs := []*filesByDir{}
-	var filesInThisDir *filesByDir
-	commitThisDir := func() {
-		if filesInThisDir != nil {
-			filesInAllDirs = append(filesInAllDirs, filesInThisDir)
-			filesInThisDir = nil
+	allDirs := []*filesByDir{}
+	var currDir *filesByDir
+	commitDir := func() {
+		if currDir != nil {
+			allDirs = append(allDirs, currDir)
+			currDir = nil
 		}
 	}
 
@@ -100,58 +108,57 @@ func getFilesByDir(box *packr.Box) []*filesByDir {
 	// all files under each first level directory appear
 	// contiguously.
 	allFiles := box.List()
-	previousDir := ""
-	for _, oneFile := range allFiles {
-		dir := strings.Split(oneFile, string(os.PathSeparator))[0]
-		if dir != previousDir {
-			commitThisDir()
-			filesInThisDir = &filesByDir{directory: dir}
+	prevName := ""
+	for _, file := range allFiles {
+		name := strings.Split(file, string(os.PathSeparator))[0]
+		if name != prevName {
+			commitDir()
+			currDir = &filesByDir{directory: name}
 		}
-		previousDir = dir
-		filesInThisDir.files = append(filesInThisDir.files, oneFile[len(dir)+len(string(os.PathSeparator)):])
+		prevName = name
+		currDir.files = append(currDir.files, file[len(name)+len(string(os.PathSeparator)):])
 	}
-	commitThisDir()
+	commitDir()
 
-	return filesInAllDirs
+	return allDirs
 }
 
-// GetMatchingFiles returns the a list of files in `box` matching
-// `srcPath`. `srcPath` should end with `os.PathSeparator` iff it refers
+// GetMatchingFiles returns the a list of files in `box` matching the
+// paths `src`. `src` should end with `os.PathSeparator` iff it refers
 // to a directory.
 //
 // As a convenience, this also returns an additional value, useful
 // for copying files within matched directories:
 //
-//  * `replacePath`: a copy of `dstPath` (which is assumed to have no
+//  * `newDst`: a copy of `dst` (which is assumed to have no
 //     trailing separator), with `os.PathSeparator` appended if
-//     `srcPath` refers to a directory
-func GetMatchingFiles(box *packr.Box, dstPath string, srcPath string) (files []string, replacePath string, err error) {
-
-	trace.Trace("reading %q", srcPath)
+//     `src` refers to a directory
+func GetMatchingFiles(box *packr.Box, dst, src string) (files []string, newDst string, err error) {
+	trace.Trace("reading %q", src)
 
-	replacePath = dstPath
+	newDst = dst
 
-	// If `srcPath` specifies a single file, match just that.
-	if !strings.HasSuffix(srcPath, string(os.PathSeparator)) {
-		if !box.Has(srcPath) {
-			err = fmt.Errorf("file box %q has no file %q", box.Name, srcPath)
+	// If `src` specifies a single file, match just that.
+	if !strings.HasSuffix(src, string(os.PathSeparator)) {
+		if !box.Has(src) {
+			err = fmt.Errorf("file box %q has no file %q", box.Name, src)
 			return
 		}
-		files = []string{srcPath}
+		files = []string{src}
 		return
 	}
 
 	// Let the caller replace the directory part of the path with
 	// the separator included.
-	replacePath = dstPath + string(os.PathSeparator)
+	newDst = dst + string(os.PathSeparator)
 
 	for _, entry := range box.List() {
-		if strings.HasPrefix(entry, srcPath) {
+		if strings.HasPrefix(entry, src) {
 			files = append(files, entry)
 		}
 	}
 	if len(files) == 0 {
-		err = fmt.Errorf("file box %q has no files matching %q", box.Name, srcPath)
+		err = fmt.Errorf("file box %q has no files matching %q", box.Name, src)
 		return
 	}
 
diff --git a/cmd/gapic-showcase/qualifier/generator.go b/cmd/gapic-showcase/qualifier/generator.go
index 3faa7f008..da0aa0201 100644
--- a/cmd/gapic-showcase/qualifier/generator.go
+++ b/cmd/gapic-showcase/qualifier/generator.go
@@ -50,23 +50,23 @@ func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]by
 
 	// Construct the various arguments to invoke the generator as a protoc plugin.
 
-	optionFlag := fmt.Sprintf("--%s_gapic_opt", gen.Language)
-	allOptions := []string{}
+	pluginOpt := fmt.Sprintf("--%s_gapic_opt", gen.Language)
+	opts := []string{}
 	if len(gen.Options) > 0 {
-		allOptions = append(allOptions, optionFlag, gen.Options)
+		opts = append(opts, pluginOpt, gen.Options)
 	}
 
-	sampleConfigFiles, _ := filesByType[fileTypeSampleConfig]
-	if len(sampleConfigFiles) > 0 {
-		allOptions = append(allOptions, optionFlag,
-			fmt.Sprintf("samples=%s", strings.Join(sampleConfigFiles, ",samples=")))
+	sampleConfigs := filesByType[fileTypeSampleConfig]
+	if len(sampleConfigs) > 0 {
+		opts = append(opts, pluginOpt,
+			fmt.Sprintf("samples=%s", strings.Join(sampleConfigs, ",samples=")))
 	}
 
 	cmdParts := []string{
 		"protoc",
 		fmt.Sprintf("--%s_gapic_out", gen.Language), fmt.Sprintf("./%s", generationDir),
 	}
-	cmdParts = append(cmdParts, allOptions...)
+	cmdParts = append(cmdParts, opts...)
 	if len(gen.Directory) > 0 {
 		cmdParts = append(cmdParts, fmt.Sprintf("--plugin=%s/protoc-gen-%s_gapic", gen.Directory, gen.Language))
 
@@ -75,8 +75,7 @@ func (gen *Generator) Run(workDir string, filesByType map[string][]string) ([]by
 
 	// Execute the command, clear all but internal errors, return.
 
-	cmdString := strings.Join(cmdParts, " ")
-	trace.Trace("running: %s", cmdString)
+	trace.Trace("running: %s", strings.Join(cmdParts, " "))
 
 	cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
 	cmd.Dir = workDir
diff --git a/cmd/gapic-showcase/qualifier/qualify.go b/cmd/gapic-showcase/qualifier/qualify.go
index 60e92bb55..624708f4f 100644
--- a/cmd/gapic-showcase/qualifier/qualify.go
+++ b/cmd/gapic-showcase/qualifier/qualify.go
@@ -35,30 +35,27 @@ func Run(settings *Settings) {
 	// wraps the current errors and includes the appropriate return code, and change main() so
 	// that if the error it gets is of type ErrorCode, it exits with that code.
 	const (
-		RetCodeSuccess = iota
-		RetCodeInternalError
-		RetCodeFailedDependencies
-		RetCodeUsageError
-		RetScenarioFailure
+		retCodeSuccess = iota
+		retCodeInternalError
+		retCodeFailedDependencies
+		retCodeUsageError
+		retCodeScenarioFailure
 	)
 
 	if err := getAssets(); err != nil {
-		os.Exit(RetCodeInternalError)
+		os.Exit(retCodeInternalError)
 	}
 
 	if err := checkDependencies(); err != nil {
-		os.Exit(RetCodeFailedDependencies)
+		os.Exit(retCodeFailedDependencies)
 	}
 
-	allScenarios, err := getTestScenarios(settings)
-	if err != nil {
-		os.Exit(RetCodeInternalError)
-	}
+	allScenarios := getTestScenarios(settings)
 
 	success := true
 	for _, scenario := range allScenarios {
 		if err := scenario.Run(); err != nil {
-			os.Exit(RetCodeInternalError)
+			os.Exit(retCodeInternalError)
 		}
 		status := "SUCCESS"
 		if !scenario.Success() {
@@ -68,16 +65,16 @@ func Run(settings *Settings) {
 		fmt.Printf("scenario %q: %s    %s\n", scenario.name, status, scenario.sandbox)
 	}
 	if !success {
-		os.Exit(RetScenarioFailure)
+		os.Exit(retCodeScenarioFailure)
 	}
 }
 
 // getTestScenarios returns a list of Scenario as found in the acceptanceSuite.
-func getTestScenarios(settings *Settings) ([]*Scenario, error) {
-	allScenarios := []*Scenario{}
-	allScenarioConfigs := getFilesByDir(acceptanceSuite)
-	for _, config := range allScenarioConfigs {
-		newScenario := &Scenario{
+func getTestScenarios(settings *Settings) []*Scenario {
+	scenarios := []*Scenario{}
+	configDirs := getFilesByDir(acceptanceSuite)
+	for _, config := range configDirs {
+		scenarios = append(scenarios, &Scenario{
 			name:         config.directory,
 			timestamp:    settings.Timestamp,
 			showcasePort: settings.ShowcasePort,
@@ -85,9 +82,9 @@ func getTestScenarios(settings *Settings) ([]*Scenario, error) {
 			files:        config.files,
 			fileBox:      acceptanceSuite,
 			schemaBox:    schemaSuite,
-		}
-		trace.Trace("adding scenario %#v", newScenario)
-		allScenarios = append(allScenarios, newScenario)
+		})
+
 	}
-	return allScenarios, nil
+	trace.Trace("adding scenarios %#v", scenarios)
+	return scenarios
 }
diff --git a/cmd/gapic-showcase/qualifier/scenario.go b/cmd/gapic-showcase/qualifier/scenario.go
index 833534402..0762cc583 100644
--- a/cmd/gapic-showcase/qualifier/scenario.go
+++ b/cmd/gapic-showcase/qualifier/scenario.go
@@ -40,12 +40,13 @@ import (
 // include all the protos that are transitively included.
 const (
 	// In a qualifier suite directory, any file that begins with this prefix is considered an
-	// include file. The rest of the filename after the prefix is the destination filename to
-	// which the files referenced by the content of the include file will be copied.
+	// include file. The rest of the filename after the prefix is the destination file or
+	// directory name to which the files referenced by the content of the include file will be
+	// copied.
 	includeFilePrefix = "include."
 
 	// In the contents of an include file, this symbol will be treated as the directory
-	// separator. At run time, it will be replaced the appropriate os.PathSeparator.
+	// separator. At run time, it will be replaced by the appropriate os.PathSeparator.
 	includeFileDirectorySeparator = '/'
 )
 
@@ -74,7 +75,7 @@ type Scenario struct {
 func (scenario *Scenario) Success() bool {
 	return scenario.generationPassed && // the generation checks passed and...
 		(!scenario.generationProcessState.Success() || // ...either generation failed, or it succeeded with ...
-			(len(scenario.filesByType["test/samples"]) == 0 || scenario.sampleTestsPassed)) // all sample tests, if any, passing
+			(len(scenario.filesByType[fileTypeSampleTest]) == 0 || scenario.sampleTestsPassed)) // all sample tests, if any, passing.
 }
 
 // Run runs this particular acceptance scenario by creating a sandbox in which it generates
@@ -121,27 +122,15 @@ func (scenario *Scenario) RunTests() {
 	trace.Trace("RunTests (TODO)")
 }
 
-// sandboxPath returns `relativePath` re-rooted at `scenario.sandbox`.
+// sandboxPath returns `relativePath` re-rooted at `scenario.sandbox`,
+// which must have been previously set.
 func (scenario *Scenario) sandboxPath(relativePath string) string {
-	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
-		panic("scenario name not set yet when trying to get sandbox filename")
-	}
-	if len(scenario.sandbox) == 0 {
-		trace.Trace("sandbox name not set yet: relativePath==%q", relativePath)
-		panic("sandbox name not set yet when trying to get sandbox filename")
-	}
-
 	return filepath.Join(scenario.sandbox, relativePath)
 }
 
 // fileBoxPath returns relativePath re-rooted at scenario.name, so that it references an object in
 // `scenario.fileBox`.
 func (scenario *Scenario) fileBoxPath(relativePath string) string {
-	if len(scenario.name) == 0 {
-		trace.Trace("scenario name not set yet: relativePath==%q", relativePath)
-		panic("scenario name not set yet when trying to get box filename")
-	}
 	return filepath.Join(scenario.name, relativePath)
 }
 
@@ -157,71 +146,62 @@ func (scenario *Scenario) fromFileBox(relativePath string) ([]byte, error) {
 func (scenario *Scenario) classifyConfigs() (err error) {
 	scenario.filesByType = make(map[string][]string)
 	// TODO: process `include.*` files first
-	for _, thisFile := range scenario.files {
-		fileTypes, err := scenario.getFileTypes(thisFile)
+	for _, file := range scenario.files {
+		types, err := scenario.getFileTypes(file)
 		if err != nil {
 			return err
 		}
-		for _, oneType := range fileTypes {
-			similarFiles := scenario.filesByType[oneType]
-			scenario.filesByType[oneType] = append(similarFiles, thisFile)
+		for _, fType := range types {
+			similarFiles := scenario.filesByType[fType]
+			scenario.filesByType[fType] = append(similarFiles, file)
 		}
-		trace.Trace("%s: type %q", thisFile, fileTypes)
+		trace.Trace("%s: type %q", file, types)
 	}
 	return nil
 }
 
-// getFileTypes gets the various type labels for `thisFile`. The type labels are either the
+// getFileTypes gets the various type labels for `file`. The type labels are either the
 // singleton list {"include"}, or a list {"scenario", ...} that includes the various types of data
 // included in that file.
-func (scenario *Scenario) getFileTypes(thisFile string) (fileTypes []string, err error) {
+func (scenario *Scenario) getFileTypes(file string) (types []string, err error) {
 
-	for idx := strings.Index(thisFile, includeFilePrefix); idx >= 0; idx = strings.Index(thisFile[idx+1:], includeFilePrefix) {
-		if !(idx == 0 || thisFile[idx-1] == os.PathSeparator) {
-			// The directory entry does not start with
-			// this pattern, so it's not an include file.
-			continue
-		}
-		if strings.Index(thisFile[idx+1:], string(os.PathListSeparator)) != -1 {
-			// There is a sub-directory component, so this is not a file.
-			continue
-		}
-
-		// This looks like a legitimate include file. Cease classifying.
+	// Identify include files in the scenario. Include files always begin with `includeFilePrefix`
+	pathParts := strings.Split(file, string(os.PathSeparator))
+	if len(pathParts) > 0 && strings.HasPrefix(pathParts[len(pathParts)-1], includeFilePrefix) {
 		return []string{"include"}, nil
 	}
 
-	extension := filepath.Ext(thisFile)
+	extension := filepath.Ext(file)
 	switch extension {
 	case ".proto":
-		fileTypes = []string{"proto"}
+		types = []string{"proto"}
 	case ".yaml", ".yml":
-		trace.Trace("reading %s", thisFile)
-		content, err := scenario.fromFileBox(thisFile)
+		trace.Trace("reading %s", file)
+		content, err := scenario.fromFileBox(file)
 		if err != nil {
-			err := fmt.Errorf("error reading %q: %w", thisFile, err)
+			err := fmt.Errorf("error reading %q: %w", file, err)
 			trace.Trace(err)
-			return fileTypes, err
+			return types, err
 		}
-		fileTypes, err = yamlDocTypes(content)
+		types, err = yamlDocTypes(content)
 		if err != nil {
-			return fileTypes, err
+			return types, err
 		}
 	}
-	fileTypes = append(fileTypes, "scenario")
-	return fileTypes, err
+	types = append(types, "scenario")
+	return types, err
 }
 
 // yamlDocTypes returns a list of the document types encountered in the (possibly multi-document)
-// YAML payload `fileContent`.
-func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
+// YAML payload `content`.
+func yamlDocTypes(content []byte) (types []string, err error) {
 	type docSchema struct {
 		Type string
 	}
 
-	decoder := yaml.NewDecoder(bytes.NewReader(fileContent))
+	decoder := yaml.NewDecoder(bytes.NewReader(content))
 	for {
-		doc := &docSchema{Type: "(UNKNOWN)"}
+		doc := &docSchema{Type: fileTypeUnknown}
 		err = decoder.Decode(doc)
 		if err == io.EOF {
 			break
@@ -231,9 +211,9 @@ func yamlDocTypes(fileContent []byte) (docTypes []string, err error) {
 			trace.Trace(err)
 			return
 		}
-		docTypes = append(docTypes, doc.Type)
+		types = append(types, doc.Type)
 	}
-	return docTypes, nil
+	return types, nil
 }
 
 // createSandbox creates a sandbox directory within which all the generation and checks for this
@@ -245,14 +225,14 @@ func (scenario *Scenario) createSandbox() (err error) {
 		return err
 	}
 
-	err = scenario.copyFiles(scenario.filesByType["scenario"])
+	err = scenario.copyFiles(scenario.filesByType[fileTypeScenario])
 	if err != nil {
 		err = fmt.Errorf("could not copy scenario files: %w", err)
 		trace.Trace(err)
 		return err
 	}
 
-	if err = scenario.getIncludes(scenario.filesByType["include"]); err != nil {
+	if err = scenario.getIncludes(scenario.filesByType[fileTypeInclude]); err != nil {
 		err = fmt.Errorf("could not copy included schema files: %w", err)
 		trace.Trace(err)
 		return err
@@ -270,64 +250,66 @@ func (scenario *Scenario) createSandbox() (err error) {
 //
 // If there's need in the future, we could potentially grow to support fetching files or directories
 // from public repositories on the web.
-func (scenario *Scenario) getIncludes(includes []string) error {
-	for _, includeFile := range includes {
+func (scenario *Scenario) getIncludes(includeFiles []string) error {
+	for _, inclusion := range includeFiles {
 
 		// The following guards against includeFilePrefix being present elsewhere in
-		// includeFile. Otherwise, we could simply use
+		// inclusion. Otherwise, we could simply use
 		//
-		//   `dstPath := strings.Replace(includeFile, includeFilePrefix, "")`
-		prefixStartIdx := strings.LastIndex(includeFile, includeFilePrefix)
-		prefixEndIdx := prefixStartIdx + len(includeFilePrefix)
-		if prefixStartIdx < 0 || prefixEndIdx >= len(includeFile) {
-			msg := fmt.Sprintf("logic error: start %d, end %d outside of range [0, %d] for %q", prefixStartIdx, prefixEndIdx, len(includeFile), includeFile)
+		//   `dstPath := strings.Replace(inclusion, includeFilePrefix, "")`
+		prefixStartIdx := strings.LastIndex(inclusion, includeFilePrefix)
+		if prefixStartIdx < 0 {
+			msg := fmt.Sprintf("logic error: did not find prefix %q in %q", includeFilePrefix, inclusion)
 			trace.Trace(msg)
-			panic(msg)
+			return fmt.Errorf(msg)
 		}
-		dstPath := includeFile[0:prefixStartIdx] + includeFile[prefixEndIdx:]
+		prefixEndIdx := prefixStartIdx + len(includeFilePrefix)
+		dstPath := inclusion[:prefixStartIdx] + inclusion[prefixEndIdx:]
 
-		content, err := scenario.fromFileBox(includeFile)
+		content, err := scenario.fromFileBox(inclusion)
 		if err != nil {
-			err = fmt.Errorf("could not read %q: %w", includeFile, err)
+			err = fmt.Errorf("could not read %q: %w", inclusion, err)
 			trace.Trace(err)
 			return err
 		}
 
 		srcPath := strings.TrimSpace(string(content))
-		srcPath = strings.ReplaceAll(srcPath, string(includeFileDirectorySeparator), string(os.PathSeparator))
+		srcPath = strings.ReplaceAll(srcPath, string(inclusion), string(os.PathSeparator))
 		files, replacePath, err := GetMatchingFiles(scenario.schemaBox, dstPath, srcPath)
 		if err != nil {
-			err = fmt.Errorf("could not process %q: %w", includeFile, err)
+			err = fmt.Errorf("could not process %q: %w", inclusion, err)
 			trace.Trace(err)
 			return err
 		}
 
-		scenario.copyFilesReplacePath(files, scenario.schemaBox.Find, srcPath, replacePath)
+		scenario.copyFilesTo(files, scenario.schemaBox.Find, srcPath, replacePath)
 	}
 	return nil
 }
 
 // copyFiles copies the files listed in `files` from `scenario.fromFileBox` to `scenario.sandbox`.
 func (scenario *Scenario) copyFiles(files []string) error {
-	return scenario.copyFilesReplacePath(files, scenario.fromFileBox, "", "")
+	return scenario.copyFilesTo(files, scenario.fromFileBox, "", "")
 }
 
-// copyFilesReplacePaths copies the files listed in `files` from the source in `fromBox` to the
-// `scenario.sandbox`. In so doing, it replaces a leading `replaceSrc` prefix in each filename with
-// `replaceDst`.
-func (scenario *Scenario) copyFilesReplacePath(files []string, fromBox func(string) ([]byte, error), replaceSrc, replaceDst string) (err error) {
+// copyFilesTo copies the files listed in `files` from the source in `fromBox` to
+// `scenario.sandbox`. In so doing, it replaces a leading `prefix` in each filename with
+// `newPrefix`.
+func (scenario *Scenario) copyFilesTo(files []string, fromBox func(string) ([]byte, error), prefix, newPrefix string) (err error) {
 	const filePermissions = 0555
-	replace := len(replaceSrc) > 0
+	replace := len(prefix) > 0
 
-	trace.Trace("replaceSrc:%q  replaceDst:%q  len(files):%d", replaceSrc, replaceDst, len(files))
+	trace.Trace("prefix:%q  newPrefix:%q  len(files):%d", prefix, newPrefix, len(files))
 
 	for _, srcFile := range files {
 		renamedFile := srcFile
 		if replace {
-			if !strings.HasPrefix(renamedFile, replaceSrc) {
-				panic(fmt.Sprintf("%q does not begin with %q", srcFile, replaceSrc))
+			if !strings.HasPrefix(renamedFile, prefix) {
+				err := fmt.Errorf("%q does not begin with %q", srcFile, prefix)
+				trace.Trace(err)
+				return err
 			}
-			renamedFile = strings.Replace(renamedFile, replaceSrc, replaceDst, 1)
+			renamedFile = strings.Replace(renamedFile, prefix, newPrefix, 1)
 		}
 
 		renamedDir := filepath.Dir(renamedFile)
diff --git a/cmd/gapic-showcase/qualify.go b/cmd/gapic-showcase/qualify.go
index a80269098..04e6d2d20 100644
--- a/cmd/gapic-showcase/qualify.go
+++ b/cmd/gapic-showcase/qualify.go
@@ -35,9 +35,15 @@ func init() {
 		ShowcasePort: 7469,
 	}
 	qualifyCmd := &cobra.Command{
-		Use:   "qualify",
+		Use:   "qualify [language]",
 		Short: "Tests a provided GAPIC generator against an acceptance suite",
-		Args:  cobra.ExactArgs(1),
+		Long: `qualify will execute a suite of acceptance checks against the GAPIC generator for the specified language.
+This confirms that the generator behaves and emits artifacts as specified in generator requirements under
+ a variety of inputs for various types of RPCs. Each acceptance check typically attempts to generate client
+libraries and corresponding standalone samples for the Showcase "Echo" service. The generator is invoked
+ as a protoc plugin; its location  may be specified via --dir, and additional generator options
+ that are needed to successfully generate the GAPIC for this API may be specified via --options,`,
+		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			// TODO: Consider moving this to a more central place for debugging all of
 			// showcase.
@@ -63,6 +69,6 @@ func init() {
 		"options",
 		"o",
 		"",
-		"The options to pass to the generator in order to generate a GAPIC for the showcase Echo service")
+		"The options to pass to the generator in order to generate a GAPIC for the Showcase \"Echo\" service")
 
 }

From fe47602bdb5c1a04f8cd6a75ece65032927229f1 Mon Sep 17 00:00:00 2001
From: Victor Chudnovsky <vchudnov@google.com>
Date: Mon, 21 Oct 2019 12:36:22 -0700
Subject: [PATCH 39/39] Give the example includes more descriptive names

And include a README about them
---
 .../acceptance_suite/basic-check/README.txt        | 14 ++++++++++++++
 ...wdREAD => include.example.nested_included_file} |  0
 ...nclude.vcREAD => include.example.included_file} |  0
 3 files changed, 14 insertions(+)
 create mode 100644 cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/README.txt
 rename cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/{include.wdREAD => include.example.nested_included_file} (100%)
 rename cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/{include.vcREAD => include.example.included_file} (100%)

diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/README.txt b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/README.txt
new file mode 100644
index 000000000..9fd290ee3
--- /dev/null
+++ b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/README.txt
@@ -0,0 +1,14 @@
+This " basic-check" scenario also showcases some features of the
+"qualifer" acceptance harness. In particular, it shows how files and
+directories can be included at various points in the sandbox hierarchy
+by means of include files:
+
+* basic-check/include.google includes the subdirectory of api-common
+  protos needed by the echo service
+  
+* basic-check/include.example.just_a_file is just an example showing
+  how to include just a file instead of a directory
+
+* basic-chec/foo/bar/include.example.an_arbitrarily_nested_file is
+  just an example showing how to include a file or directory at any
+  point in a sandbox directory tree.
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.wdREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.example.nested_included_file
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.wdREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/foo/bar/include.example.nested_included_file
diff --git a/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.vcREAD b/cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.example.included_file
similarity index 100%
rename from cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.vcREAD
rename to cmd/gapic-showcase/qualifier/acceptance_suite/basic-check/include.example.included_file