gnostic.go

// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./COMPILE-PROTOS.sh

// Gnostic is a tool for building better REST APIs through knowledge.
//
// Gnostic reads declarative descriptions of REST APIs that conform
// to the OpenAPI Specification, reports errors, resolves internal
// dependencies, and puts the results in a binary form that can
// be used in any language that is supported by the Protocol Buffer
// tools.
//
// Gnostic models are validated and typed. This allows API tool
// developers to focus on their product and not worry about input
// validation and type checking.
//
// Gnostic calls plugins that implement a variety of API implementation
// and support features including generation of client and server
// support code.
package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/OpenAPIv3"
	"github.com/googleapis/gnostic/compiler"
	"github.com/googleapis/gnostic/discovery"
	"github.com/googleapis/gnostic/jsonwriter"
	plugins "github.com/googleapis/gnostic/plugins"
	surface "github.com/googleapis/gnostic/surface"
	"gopkg.in/yaml.v2"
)

const ( // Source Format
	SourceFormatUnknown   = 0
	SourceFormatOpenAPI2  = 2
	SourceFormatOpenAPI3  = 3
	SourceFormatDiscovery = 4
)

// Determine the version of an OpenAPI description read from JSON or YAML.
func getOpenAPIVersionFromInfo(info interface{}) int {
	m, ok := compiler.UnpackMap(info)
	if !ok {
		return SourceFormatUnknown
	}
	swagger, ok := compiler.MapValueForKey(m, "swagger").(string)
	if ok && strings.HasPrefix(swagger, "2.0") {
		return SourceFormatOpenAPI2
	}
	openapi, ok := compiler.MapValueForKey(m, "openapi").(string)
	if ok && strings.HasPrefix(openapi, "3.0") {
		return SourceFormatOpenAPI3
	}
	kind, ok := compiler.MapValueForKey(m, "kind").(string)
	if ok && kind == "discovery#restDescription" {
		return SourceFormatDiscovery
	}
	return SourceFormatUnknown
}

const (
	pluginPrefix    = "gnostic-"
	extensionPrefix = "gnostic-x-"
)

type pluginCall struct {
	Name       string
	Invocation string
}

// Invokes a plugin.
func (p *pluginCall) perform(document proto.Message, sourceFormat int, sourceName string, timePlugins bool) ([]*plugins.Message, error) {
	if p.Name != "" {
		request := &plugins.Request{}

		// Infer the name of the executable by adding the prefix.
		executableName := pluginPrefix + p.Name

		// Validate invocation string with regular expression.
		invocation := p.Invocation

		//
		// Plugin invocations must consist of
		// zero or more comma-separated key=value pairs followed by a path.
		// If pairs are present, a colon separates them from the path.
		// Keys and values must be alphanumeric strings and may contain
		// dashes, underscores, periods, or forward slashes.
		// A path can contain any characters other than the separators ',', ':', and '='.
		//
		invocationRegex := regexp.MustCompile(`^([\w-_\/\.]+=[\w-_\/\.]+(,[\w-_\/\.]+=[\w-_\/\.]+)*:)?[^,:=]+$`)
		if !invocationRegex.Match([]byte(p.Invocation)) {
			return nil, fmt.Errorf("Invalid invocation of %s: %s", executableName, invocation)
		}

		invocationParts := strings.Split(p.Invocation, ":")
		var outputLocation string
		switch len(invocationParts) {
		case 1:
			outputLocation = invocationParts[0]
		case 2:
			parameters := strings.Split(invocationParts[0], ",")
			for _, keyvalue := range parameters {
				pair := strings.Split(keyvalue, "=")
				if len(pair) == 2 {
					request.Parameters = append(request.Parameters, &plugins.Parameter{Name: pair[0], Value: pair[1]})
				}
			}
			outputLocation = invocationParts[1]
		default:
			// badly-formed request
			outputLocation = invocationParts[len(invocationParts)-1]
		}

		version := &plugins.Version{}
		version.Major = 0
		version.Minor = 1
		version.Patch = 0
		request.CompilerVersion = version

		request.OutputPath = outputLocation

		request.SourceName = sourceName
		switch sourceFormat {
		case SourceFormatOpenAPI2:
			request.AddModel("openapi.v2.Document", document)
			// include experimental API surface model
			surfaceModel, err := surface.NewModelFromOpenAPI2(document.(*openapi_v2.Document))
			if err == nil {
				request.AddModel("surface.v1.Model", surfaceModel)
			}
		case SourceFormatOpenAPI3:
			request.AddModel("openapi.v3.Document", document)
			// include experimental API surface model
			surfaceModel, err := surface.NewModelFromOpenAPI3(document.(*openapi_v3.Document))
			if err == nil {
				request.AddModel("surface.v1.Model", surfaceModel)
			}
		case SourceFormatDiscovery:
			request.AddModel("discovery.v1.Document", document)
		default:
		}

		requestBytes, _ := proto.Marshal(request)

		cmd := exec.Command(executableName, "-plugin")
		cmd.Stdin = bytes.NewReader(requestBytes)
		cmd.Stderr = os.Stderr
		pluginStartTime := time.Now()
		output, err := cmd.Output()
		pluginElapsedTime := time.Since(pluginStartTime)
		if timePlugins {
			fmt.Printf("> %s (%s)\n", executableName, pluginElapsedTime)
		}
		if err != nil {
			return nil, err
		}
		response := &plugins.Response{}
		err = proto.Unmarshal(output, response)
		if err != nil {
			// Gnostic expects plugins to only write the
			// response message to stdout. Be sure that
			// any logging messages are written to stderr only.
			return nil, errors.New("Invalid plugin response (plugins must write log messages to stderr, not stdout).")
		}

		err = plugins.HandleResponse(response, outputLocation)

		return response.Messages, err
	}
	return nil, nil
}

func isFile(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !fileInfo.IsDir()
}

func isDirectory(path string) bool {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fileInfo.IsDir()
}

// Write bytes to a named file.
// Certain names have special meaning:
//   ! writes nothing
//   - writes to stdout
//   = writes to stderr
// If a directory name is given, the file is written there with
// a name derived from the source and extension arguments.
func writeFile(name string, bytes []byte, source string, extension string) {
	var writer io.Writer
	if name == "!" {
		return
	} else if name == "-" {
		writer = os.Stdout
	} else if name == "=" {
		writer = os.Stderr
	} else if isDirectory(name) {
		base := filepath.Base(source)
		// Remove the original source extension.
		base = base[0 : len(base)-len(filepath.Ext(base))]
		// Build the path that puts the result in the passed-in directory.
		filename := name + "/" + base + "." + extension
		file, _ := os.Create(filename)
		defer file.Close()
		writer = file
	} else {
		file, _ := os.Create(name)
		defer file.Close()
		writer = file
	}
	writer.Write(bytes)
	if name == "-" || name == "=" {
		writer.Write([]byte("\n"))
	}
}

// The Gnostic structure holds global state information for gnostic.
type Gnostic struct {
	usage             string
	sourceName        string
	binaryOutputPath  string
	textOutputPath    string
	yamlOutputPath    string
	jsonOutputPath    string
	errorOutputPath   string
	messageOutputPath string
	resolveReferences bool
	pluginCalls       []*pluginCall
	extensionHandlers []compiler.ExtensionHandler
	sourceFormat      int
	timePlugins       bool
}

// Initialize a structure to store global application state.
func newGnostic() *Gnostic {
	g := &Gnostic{}
	// Option fields initialize to their default values.
	g.usage = `
Usage: gnostic SOURCE [OPTIONS]
  SOURCE is the filename or URL of an API description.
Options:
  --pb-out=PATH       Write a binary proto to the specified location.
  --text-out=PATH     Write a text proto to the specified location.
  --json-out=PATH     Write a json API description to the specified location.
  --yaml-out=PATH     Write a yaml API description to the specified location.
  --errors-out=PATH   Write compilation errors to the specified location.
  --messages-out=PATH Write messages generated by plugins to the specified
                      location. Messages from all plugin invocations are
                      written to a single common file.
  --PLUGIN-out=PATH   Run the plugin named gnostic-PLUGIN and write results
                      to the specified location.
  --PLUGIN            Run the plugin named gnostic-PLUGIN but don't write any
                      results. Used for plugins that return messages only.
                      PLUGIN must not match any other gnostic option.
  --x-EXTENSION       Use the extension named gnostic-x-EXTENSION
                      to process OpenAPI specification extensions.
  --resolve-refs      Explicitly resolve $ref references.
                      This could have problems with recursive definitions.
  --time-plugins      Report plugin runtimes.
`
	// Initialize internal structures.
	g.pluginCalls = make([]*pluginCall, 0)
	g.extensionHandlers = make([]compiler.ExtensionHandler, 0)
	return g
}

// Parse command-line options.
func (g *Gnostic) readOptions() {
	// plugin processing matches patterns of the form "--PLUGIN-out=PATH" and "--PLUGIN_out=PATH"
	pluginRegex := regexp.MustCompile("--(.+)[-_]out=(.+)")

	// extension processing matches patterns of the form "--x-EXTENSION"
	extensionRegex := regexp.MustCompile("--x-(.+)")

	for i, arg := range os.Args {
		if i == 0 {
			continue // skip the tool name
		}
		var m [][]byte
		if m = pluginRegex.FindSubmatch([]byte(arg)); m != nil {
			pluginName := string(m[1])
			invocation := string(m[2])
			switch pluginName {
			case "pb":
				g.binaryOutputPath = invocation
			case "text":
				g.textOutputPath = invocation
			case "json":
				g.jsonOutputPath = invocation
			case "yaml":
				g.yamlOutputPath = invocation
			case "errors":
				g.errorOutputPath = invocation
			case "messages":
				g.messageOutputPath = invocation
			default:
				p := &pluginCall{Name: pluginName, Invocation: invocation}
				g.pluginCalls = append(g.pluginCalls, p)
			}
		} else if m = extensionRegex.FindSubmatch([]byte(arg)); m != nil {
			extensionName := string(m[1])
			extensionHandler := compiler.ExtensionHandler{Name: extensionPrefix + extensionName}
			g.extensionHandlers = append(g.extensionHandlers, extensionHandler)
		} else if arg == "--resolve-refs" {
			g.resolveReferences = true
		} else if arg == "--time-plugins" {
			g.timePlugins = true
		} else if arg[0] == '-' && arg[1] == '-' {
			// try letting the option specify a plugin with no output files (or unwanted output files)
			// this is useful for calling plugins like linters that only return messages
			p := &pluginCall{Name: arg[2:len(arg)], Invocation: "!"}
			g.pluginCalls = append(g.pluginCalls, p)
		} else if arg[0] == '-' {
			fmt.Fprintf(os.Stderr, "Unknown option: %s.\n%s\n", arg, g.usage)
			os.Exit(-1)
		} else {
			g.sourceName = arg
		}
	}
}

// Validate command-line options.
func (g *Gnostic) validateOptions() {
	if g.binaryOutputPath == "" &&
		g.textOutputPath == "" &&
		g.yamlOutputPath == "" &&
		g.jsonOutputPath == "" &&
		g.errorOutputPath == "" &&
		len(g.pluginCalls) == 0 {
		fmt.Fprintf(os.Stderr, "Missing output directives.\n%s\n", g.usage)
		os.Exit(-1)
	}
	if g.sourceName == "" {
		fmt.Fprintf(os.Stderr, "No input specified.\n%s\n", g.usage)
		os.Exit(-1)
	}
	// If we get here and the error output is unspecified, write errors to stderr.
	if g.errorOutputPath == "" {
		g.errorOutputPath = "="
	}
}

// Generate an error message to be written to stderr or a file.
func (g *Gnostic) errorBytes(err error) []byte {
	return []byte("Errors reading " + g.sourceName + "\n" + err.Error())
}

// Read an OpenAPI description from YAML or JSON.
func (g *Gnostic) readOpenAPIText(bytes []byte) (message proto.Message, err error) {
	info, err := compiler.ReadInfoFromBytes(g.sourceName, bytes)
	if err != nil {
		return nil, err
	}
	// Determine the OpenAPI version.
	g.sourceFormat = getOpenAPIVersionFromInfo(info)
	if g.sourceFormat == SourceFormatUnknown {
		return nil, errors.New("unable to identify OpenAPI version")
	}
	// Compile to the proto model.
	if g.sourceFormat == SourceFormatOpenAPI2 {
		document, err := openapi_v2.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
		if err != nil {
			return nil, err
		}
		message = document
	} else if g.sourceFormat == SourceFormatOpenAPI3 {
		document, err := openapi_v3.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
		if err != nil {
			return nil, err
		}
		message = document
	} else {
		document, err := discovery_v1.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
		if err != nil {
			return nil, err
		}
		message = document
	}
	return message, err
}

// Read an OpenAPI binary file.
func (g *Gnostic) readOpenAPIBinary(data []byte) (message proto.Message, err error) {
	// try to read an OpenAPI v3 document
	documentV3 := &openapi_v3.Document{}
	err = proto.Unmarshal(data, documentV3)
	if err == nil && strings.HasPrefix(documentV3.Openapi, "3.0") {
		g.sourceFormat = SourceFormatOpenAPI3
		return documentV3, nil
	}
	// if that failed, try to read an OpenAPI v2 document
	documentV2 := &openapi_v2.Document{}
	err = proto.Unmarshal(data, documentV2)
	if err == nil && strings.HasPrefix(documentV2.Swagger, "2.0") {
		g.sourceFormat = SourceFormatOpenAPI2
		return documentV2, nil
	}
	// if that failed, try to read a Discovery Format document
	discoveryDocument := &discovery_v1.Document{}
	err = proto.Unmarshal(data, discoveryDocument)
	if err == nil { // && strings.HasPrefix(documentV2.Swagger, "2.0") {
		g.sourceFormat = SourceFormatDiscovery
		return discoveryDocument, nil
	}
	return nil, err
}

// Write a binary pb representation.
func (g *Gnostic) writeBinaryOutput(message proto.Message) {
	protoBytes, err := proto.Marshal(message)
	if err != nil {
		writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
		defer os.Exit(-1)
	} else {
		writeFile(g.binaryOutputPath, protoBytes, g.sourceName, "pb")
	}
}

// Write a text pb representation.
func (g *Gnostic) writeTextOutput(message proto.Message) {
	bytes := []byte(proto.MarshalTextString(message))
	writeFile(g.textOutputPath, bytes, g.sourceName, "text")
}

// Write JSON/YAML OpenAPI representations.
func (g *Gnostic) writeJSONYAMLOutput(message proto.Message) {
	// Convert the OpenAPI document into an exportable MapSlice.
	var rawInfo yaml.MapSlice
	var ok bool
	var err error
	if g.sourceFormat == SourceFormatOpenAPI2 {
		document := message.(*openapi_v2.Document)
		rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
		if !ok {
			rawInfo = nil
		}
	} else if g.sourceFormat == SourceFormatOpenAPI3 {
		document := message.(*openapi_v3.Document)
		rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
		if !ok {
			rawInfo = nil
		}
	} else if g.sourceFormat == SourceFormatDiscovery {
		document := message.(*discovery_v1.Document)
		rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
		if !ok {
			rawInfo = nil
		}
	}
	// Optionally write description in yaml format.
	if g.yamlOutputPath != "" {
		var bytes []byte
		if rawInfo != nil {
			bytes, err = yaml.Marshal(rawInfo)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error generating yaml output %s\n", err.Error())
			}
			writeFile(g.yamlOutputPath, bytes, g.sourceName, "yaml")
		} else {
			fmt.Fprintf(os.Stderr, "No yaml output available.\n")
		}
	}
	// Optionally write description in json format.
	if g.jsonOutputPath != "" {
		var bytes []byte
		if rawInfo != nil {
			bytes, _ = jsonwriter.Marshal(rawInfo)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error generating json output %s\n", err.Error())
			}
			writeFile(g.jsonOutputPath, bytes, g.sourceName, "json")
		} else {
			fmt.Fprintf(os.Stderr, "No json output available.\n")
		}
	}
}

// Write messages.
func (g *Gnostic) writeMessagesOutput(message proto.Message) {
	protoBytes, err := proto.Marshal(message)
	if err != nil {
		writeFile(g.messageOutputPath, g.errorBytes(err), g.sourceName, "errors")
		defer os.Exit(-1)
	} else {
		writeFile(g.messageOutputPath, protoBytes, g.sourceName, "messages.pb")
	}
}

// Perform all actions specified in the command-line options.
func (g *Gnostic) performActions(message proto.Message) (err error) {
	// Optionally resolve internal references.
	if g.resolveReferences {
		if g.sourceFormat == SourceFormatOpenAPI2 {
			document := message.(*openapi_v2.Document)
			_, err = document.ResolveReferences(g.sourceName)
		} else if g.sourceFormat == SourceFormatOpenAPI3 {
			document := message.(*openapi_v3.Document)
			_, err = document.ResolveReferences(g.sourceName)
		}
		if err != nil {
			return err
		}
	}
	// Optionally write proto in binary format.
	if g.binaryOutputPath != "" {
		g.writeBinaryOutput(message)
	}
	// Optionally write proto in text format.
	if g.textOutputPath != "" {
		g.writeTextOutput(message)
	}
	// Optionally write document in yaml and/or json formats.
	if g.yamlOutputPath != "" || g.jsonOutputPath != "" {
		g.writeJSONYAMLOutput(message)
	}
	// Call all specified plugins.
	messages := make([]*plugins.Message, 0)
	for _, p := range g.pluginCalls {
		pluginMessages, err := p.perform(message, g.sourceFormat, g.sourceName, g.timePlugins)
		if err != nil {
			writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
			defer os.Exit(-1) // run all plugins, even when some have errors
		}
		messages = append(messages, pluginMessages...)
	}
	if g.messageOutputPath != "" {
		g.writeMessagesOutput(&plugins.Messages{Messages: messages})
	} else {
		// Print any messages from the plugins
		if len(messages) > 0 {
			for _, message := range messages {
				fmt.Printf("%+v\n", message)
			}
		}
	}
	return nil
}

func (g *Gnostic) main() {
	var err error
	g.readOptions()
	g.validateOptions()
	// Read the OpenAPI source.
	bytes, err := compiler.ReadBytesForFile(g.sourceName)
	if err != nil {
		writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
		os.Exit(-1)
	}
	extension := strings.ToLower(filepath.Ext(g.sourceName))
	var message proto.Message
	if extension == ".json" || extension == ".yaml" {
		// Try to read the source as JSON/YAML.
		message, err = g.readOpenAPIText(bytes)
		if err != nil {
			writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
			os.Exit(-1)
		}
	} else if extension == ".pb" {
		// Try to read the source as a binary protocol buffer.
		message, err = g.readOpenAPIBinary(bytes)
		if err != nil {
			writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
			os.Exit(-1)
		}
	} else {
		err = errors.New("unknown file extension. 'json', 'yaml', and 'pb' are accepted")
		writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
		os.Exit(-1)
	}
	// Perform actions specified by command options.
	err = g.performActions(message)
	if err != nil {
		writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
		os.Exit(-1)
	}
}

func main() {
	g := newGnostic()
	g.main()
}