Merge pull request #762 from radu-matei/generic-webhook-cleanup

Add generic gateway

.gitignore

@@ -4,6 +4,7 @@
 /brigade-controller/rootfs/brigade-controller
 /brigade-cr-gateway/rootfs/brigade-cr-gateway
 /brigade-github-gateway/rootfs/brigade-github-gateway
+/brigade-generic-gateway/rootfs/brigade-generic-gateway
 /brigade-vacuum/rootfs/brigade-vacuum
 /brigade-worker/dist/
 /brigade-worker/doc/

Makefile

@@ -5,14 +5,16 @@ DOCKER_REGISTRY    ?= deis
 DOCKER_BUILD_FLAGS :=
 LDFLAGS            :=
 
+
 # Helm chart/release defaults
 BRIGADE_RELEASE            ?= brigade-server
 BRIGADE_NAMESPACE          ?= default
 BRIGADE_GITHUB_GW_SERVICE  := $(BRIGADE_RELEASE)-brigade-github-gw
 BRIGADE_GITHUB_GW_PORT     := 7744
 
-BINS        = brigade-api brigade-controller brigade-github-gateway brigade-cr-gateway brigade-vacuum brig
-IMAGES      = brigade-api brigade-controller brigade-github-gateway brigade-cr-gateway brigade-vacuum brig brigade-worker git-sidecar
+BINS        = brigade-api brigade-controller brigade-github-gateway brigade-cr-gateway brigade-generic-gateway brigade-vacuum brig
+IMAGES      = brigade-api brigade-controller brigade-github-gateway brigade-cr-gateway brigade-generic-gateway brigade-vacuum brig brigade-worker git-sidecar
+
 
 .PHONY: echo-images
 echo-images:

brig/cmd/brig/commands/project_create.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/Masterminds/goutils"
@@ -529,9 +530,40 @@ func projectAdvancedPrompts(p *brigade.Project, store storage.Store) error {
 		p.DefaultScript = loadFileStr(fname)
 	}
 
+	err = survey.AskOne(&survey.Input{
+		Message: "Secret for the Generic Gateway (alphanumeric characters only). Press Enter if you want it to be auto-generated",
+		Help:    "This is the secret that secures the Generic Gateway. Only alphanumeric characters are accepted. Provide an empty string if you want an auto-generated one",
+	}, &p.GenericGatewaySecret, genericGatewaySecretValidator)
+	if err != nil {
+		return fmt.Errorf(abort, err)
+	}
+
+	// user pressed Enter key, so let's auto-generate a GenericGateway secret
+	if p.GenericGatewaySecret == "" {
+		var err error
+		p.GenericGatewaySecret, err = goutils.RandomAlphaNumeric(5)
+		if err != nil {
+			return fmt.Errorf("Error in generating Generic Gateway Secret: %s", err.Error())
+		}
+		fmt.Printf("Auto-generated Generic Gateway Secret: %s\n", p.GenericGatewaySecret)
+	}
+
 	return nil
 }
 
+// genericGatewaySecretValidator validates the secret provided by user for the Generic Gateway
+// this can be either "" (so it will be auto-generated) or alphanumeric
+func genericGatewaySecretValidator(val interface{}) error {
+	re := regexp.MustCompile("^[a-zA-Z0-9]*$")
+	if val.(string) == "" {
+		return nil
+	}
+	if re.MatchString(val.(string)) {
+		return nil
+	}
+	return fmt.Errorf("Generic Gateway secret should only contain alphanumeric characters")
+}
+
 // loadFileValidator validates that a file exists and can be read.
 func loadFileValidator(val interface{}) error {
 	name := os.ExpandEnv(val.(string))

brig/cmd/brig/commands/project_create_test.go

@@ -101,3 +101,26 @@ func TestReplaceNewlines(t *testing.T) {
 		t.Fatalf("Expected %q, got %q", expect, got)
 	}
 }
+
+func TestGenericGatewaySecretValidator(t *testing.T) {
+	s1 := "asdf"
+	if err := genericGatewaySecretValidator(s1); err != nil {
+		t.Fatal("Expected nil, got error")
+	}
+	s2 := "AsDf1"
+	if err := genericGatewaySecretValidator(s2); err != nil {
+		t.Fatal("Expected nil, got error")
+	}
+	s3 := "jfdkfd^&"
+	if err := genericGatewaySecretValidator(s3); err == nil {
+		t.Fatal("Expected error, got nil")
+	}
+	s4 := ""
+	if err := genericGatewaySecretValidator(s4); err != nil {
+		t.Fatal("Expected nil, got error")
+	}
+	s5 := " "
+	if err := genericGatewaySecretValidator(s5); err == nil {
+		t.Fatal("Expected error, got nil")
+	}
+}

brigade-cr-gateway/README.md

@@ -13,5 +13,5 @@ If you do not see your preferred registry above, you can do any of the following
   us.
 2. Write a custom gateway. This may be necessary if your chosen container registry
   uses even moderately exotic auth patterns.
-3. File an issue in the issue queue here, and see if you can rally some support
+3. File an issue in the issue queue [here](https://github.com/Azure/brigade/issues), and see if you can rally some support
   for building one.

brigade-generic-gateway/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.8
+
+RUN apk update && apk add --no-cache \
+    ca-certificates \
+    git \
+    && update-ca-certificates
+
+COPY rootfs/brigade-generic-gateway /usr/bin/brigade-generic-gateway
+
+CMD /usr/bin/brigade-generic-gateway

brigade-generic-gateway/README.md

@@ -0,0 +1,3 @@
+# Brigade Generic Gateway
+
+This server provides a generic gateway. You can check [here](https://azure.github.io/brigade/topics/genericgateway.html) for the relevant documentation.

brigade-generic-gateway/cmd/brigade-generic-gateway/server.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"os"
+
+	gin "gopkg.in/gin-gonic/gin.v1"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/Azure/brigade/pkg/storage"
+	"github.com/Azure/brigade/pkg/storage/kube"
+	"github.com/Azure/brigade/pkg/webhook"
+)
+
+var (
+	kubeconfig string
+	master     string
+	namespace  string
+)
+
+func init() {
+	flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file")
+	flag.StringVar(&master, "master", "", "master url")
+	flag.StringVar(&namespace, "namespace", defaultNamespace(), "kubernetes namespace")
+}
+
+func main() {
+	flag.Parse()
+
+	clientset, err := kube.GetClient(master, kubeconfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if namespace == "" {
+		namespace = v1.NamespaceDefault
+	}
+
+	store := kube.New(clientset, namespace)
+
+	router := newRouter(store)
+	router.Run(":8000")
+}
+
+func newRouter(store storage.Store) *gin.Engine {
+	router := gin.New()
+	router.Use(gin.Recovery())
+
+	handler := webhook.NewGenericWebhook(store)
+
+	events := router.Group("/webhook")
+	{
+		events.Use(gin.Logger())
+		events.POST("/:projectID/:secret", handler)
+	}
+
+	router.GET("/healthz", healthz)
+	return router
+}
+
+func healthz(c *gin.Context) {
+	c.String(http.StatusOK, http.StatusText(http.StatusOK))
+}
+
+func defaultNamespace() string {
+	if ns, ok := os.LookupEnv("BRIGADE_NAMESPACE"); ok {
+		return ns
+	}
+	return v1.NamespaceDefault
+}

brigade-generic-gateway/cmd/brigade-generic-gateway/server_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/Azure/brigade/pkg/storage/mock"
+)
+
+func TestNewRouter(t *testing.T) {
+	s := mock.New()
+	s.ProjectList[0].ID = "brigade-4625a05cf6914e556aa254cb2af234203744de2f"
+	s.ProjectList[0].Name = "deis/empty-testbed"
+	s.ProjectList[0].GenericGatewaySecret = "mysecret"
+	r := newRouter(s)
+
+	if r == nil {
+		t.Fail()
+	}
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	res, err := http.Get(ts.URL + "/healthz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 200 {
+		t.Fatalf("Unexpected status on healthz: %s", res.Status)
+	}
+
+	body, err := ioutil.ReadFile("./testdata/genericwebhook.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	route400 := "/webhook/brigade-4625a05cf6914e556aa254cb2af234203744de2f_WRONG_URL/mysecret"
+	res, err = http.Post(ts.URL+route400, "application/json", bytes.NewBuffer(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 400 {
+		t.Fatalf("Expected 400 status, got: %s", res.Status)
+	}
+
+	route401 := "/webhook/brigade-4625a05cf6914e556aa254cb2af234203744de2f/mysecret2"
+	res, err = http.Post(ts.URL+route401, "application/json", bytes.NewBuffer(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 401 {
+		t.Fatalf("Expected 401 status, got: %s", res.Status)
+	}
+
+	corruptbody, err := ioutil.ReadFile("./testdata/genericwebhook.json.corrupt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	routeCorruptBody := "/webhook/brigade-4625a05cf6914e556aa254cb2af234203744de2f/mysecret"
+	res, err = http.Post(ts.URL+routeCorruptBody, "application/json", bytes.NewBuffer(corruptbody))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 400 {
+		t.Fatalf("Expected 400 status, got: %s", res.Status)
+	}
+
+}

brigade-generic-gateway/cmd/brigade-generic-gateway/testdata/genericwebhook.json

@@ -0,0 +1,4 @@
+{
+	"ref": "0.2",
+	"commit": "1e175a8"
+}

brigade-generic-gateway/cmd/brigade-generic-gateway/testdata/genericwebhook.json.corrupt

@@ -0,0 +1,5 @@
+{
+	"ref": "0.2",
+	"commit": "1e175a8"
+	BLABLABLA
+}

docs/topics/genericgateway.md

@@ -0,0 +1,67 @@
+# Generic Gateway
+
+Brigade contains a Generic Gateway that can be used to accept requests from other platforms or systems. Generic Gateway is a separate component in the Brigade system, like Github and Container Registry (CR) Gateways.
+
+Generic Gateway is _not enabled by default_.
+
+## Intro to Generic Gateway 
+
+Generic Gateway can optionally be activated to accept `POST` webhook requests at `/webhook/:projectID/:secret` path. When this endpoint is called, Brigade will respond by creating a Build with a `webhook` event. This provides Brigade developers with the ability to trigger scripts based on messages received from any platform that can send a POST HTTP request.
+
+## Generic Gateway
+
+Generic Gateway is disabled by default, but can easily be turned on during installation or upgrade of Brigade:
+
+```
+$ helm install -n brigade brigade/brigade --set genericGateway.enabled=true
+```
+
+This will enable the Generic Gateway Deployment/Services/RBAC permissions. However, be aware that Generic Gateway is not exposed outside the cluster. In case you are certain from a security perspective that you want to expose the Generic Gateway via a [Kubernetes LoadBalancer Service](https://kubernetes.io/docs/concepts/services-networking/#loadbalancer), you can install Brigade via the following command:
+
+```
+$ helm install -n brigade brigade/brigade --set genericGateway.enabled=true,genericGateway.service.type=LoadBalancer
+```
+
+Alternatively, for enhanced security, you can install an SSL proxy (like `cert-manager`) and direct it to the Generic Gateway Service.
+
+## Using the Generic Gateway
+
+As mentioned, Generic Gateway accepts POST requests at `/webhook/:projectID/:secret` endpoint. These requests should also carry a JSON payload.
+
+- `projectID` is the Brigade Project ID
+- `secret` is a custom secret for this specific project's Generic Gateway webhook support. In other words, each project that wants to accept Generic Gateway events should have its own Generic Gateway secret. This secret serves as a simple authentication mechanism.
+
+When you create a new Brigade Project via Brig CLI, you can optionally create such a secret by using the Advanced Options during `brig project create`. This secret must contain only alphanumeric characters. If you provide an empty string, Brig will generate and output a secret for you.
+
+*Important*: If you do not go into "Advanced Options" during `brig project create`, a secret will not be created and you will not be able to use Generic Gateway for your project. However, you can always use `brig project create --replace` (or just `kubectl edit` your project Secret) to update your project and include a `genericGatewaySecret` string value.
+
+When calling the Generic Gateway webhook endpoint, you can include a custom JSON payload such as:
+
+```json
+{
+	"ref": "refs/heads/changes",
+	"commit": "b60ad9543b2ddbbe73430dd6898b75883306cecc"
+}
+```
+
+`Ref` and `commit` values would be used to configure the specific revision that Brigade will pull from your repository.
+
+Last but not least, here is a sample Brigade.js file that could be used as a base for your own scripts that respond to Generic Gateway's `webhook` event. This script will echo the name of your project and 'webhook'.
+
+```javascript
+const { events, Job } = require("brigadier");
+events.on("webhook", (e, p) => {
+  var echo = new Job("echo", "alpine:3.8");
+  echo.storage.enabled = false;
+  echo.tasks = [
+    "echo Project " + p.name,
+    "echo Event $EVENT_NAME"
+  ];
+
+  echo.env = {
+    "EVENT_NAME": e.type
+  };
+
+  echo.run();
+});
+```

docs/topics/index.md

@@ -14,6 +14,7 @@ If you don't see a topic guide here and have a reasonable level of knowledge on
   - [Scripting Guide - Advanced](scripting_advanced.md): Advanced examples for `brigade.js` files.
   - [GitHub Integration](github.md): A guide for configuring GitHub integration.
   - [Container Registry Integration](dockerhub.md): A guide for configuring integration with DockerHub or Azure Container Registry.
+  - [Generic Gateway](genericgateway.md): How to use Brigade's Generic Gateway functionality.
   - [Using Secrets](secrets.md): How to pass sensitive data into builds.
   - [Brigade Gateways](gateways.md): Learn how to write your own Brigade gateway.
 - Configuring and Running Brigade

pkg/brigade/project.go

@@ -51,6 +51,8 @@ type Project struct {
 
 	// BrigadejsPath contains the path for the Brigade.js file in the source repo
 	BrigadejsPath string `json:"brigadejsPath"`
+	// GenericGatewaySecret is a string that contains the access code used by API Server to authenticate generic Gateway requests
+	GenericGatewaySecret string `json:"genericGatewaySecret"`
 }
 
 // SecretsMap is a map[string]string for storing secrets.