From e56598d2843c2c58b6b798bf862a3ea0b11caac7 Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 30 Jan 2019 11:15:40 +0100 Subject: [PATCH 1/6] Enable flagger in production not staging --- flagger/README.md | 17 ++++++++++++++--- flagger/croc-hunter-canary.yaml | 8 ++++---- istio/README.md | 5 +++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/flagger/README.md b/flagger/README.md index 45042aa..63fabb1 100644 --- a/flagger/README.md +++ b/flagger/README.md @@ -2,12 +2,23 @@ Install [Flagger](https://docs.flagger.app/install/install-flagger) -Enable Istio in the jx-staging and jx-production namespaces +Enable Istio in the `jx-staging` and `jx-production` namespaces for metrics gathering - kubectl patch ns jx-staging --type=json -p='[{"op": "add", "path": "/metadata/labels/istio-injection", "value": "enabled"}]' - kubectl patch ns jx-production --type=json -p='[{"op": "add", "path": "/metadata/labels/istio-injection", "value": "enabled"}]' + kubectl label namespace jx-staging istio-injection=enabled + kubectl label namespace jx-production istio-injection=enabled Create the canary object that will add our deployment to Flagger kubectl create -f croc-hunter-canary.yaml + +Optional: Create a `ServiceEntry` to allow traffic to the Google metadata api to display the region + + kubectl create -f ../istio/google-api.yaml + +# Grafana dashboard + + kubectl --namespace istio-system port-forward deploy/flagger-grafana 3000 + kubectl --namespace istio-system port-forward deploy/prometheus 9090 + +[http://localhost:3000](http://localhost:3000) diff --git a/flagger/croc-hunter-canary.yaml b/flagger/croc-hunter-canary.yaml index e9197e3..b02a8fa 100644 --- a/flagger/croc-hunter-canary.yaml +++ b/flagger/croc-hunter-canary.yaml @@ -2,19 +2,19 @@ apiVersion: flagger.app/v1alpha2 kind: Canary metadata: # canary name must match deployment name - name: jx-staging-croc-hunter-jenkinsx - namespace: jx-staging + name: jx-production-croc-hunter-jenkinsx + namespace: jx-production spec: # deployment reference targetRef: apiVersion: apps/v1 kind: Deployment - name: jx-staging-croc-hunter-jenkinsx + name: jx-production-croc-hunter-jenkinsx # HPA reference (optional) # autoscalerRef: # apiVersion: autoscaling/v2beta1 # kind: HorizontalPodAutoscaler - # name: jx-staging-croc-hunter-jenkinsx + # name: jx-production-croc-hunter-jenkinsx # the maximum time in seconds for the canary deployment # to make progress before it is rollback (default 600s) progressDeadlineSeconds: 60 diff --git a/istio/README.md b/istio/README.md index a8b1024..376a72d 100644 --- a/istio/README.md +++ b/istio/README.md @@ -4,6 +4,7 @@ Install [Istio](https://istio.io/docs/setup/kubernetes/quick-start/) ## Access from the Internet using Istio ingress gateway +Istio will route the traffic entering through the ingress gateway. Find the [ingress gateway ip address](https://istio.io/docs/tasks/traffic-management/ingress/#determining-the-ingress-ip-and-ports) and configure a wildcard DNS for it. For example map `*.example.com` to @@ -24,8 +25,8 @@ If you need to access the service through Istio from inside the cluster (not nee Enable Istio in the jx-staging and jx-production namespaces - kubectl patch ns jx-carlossg-croc-hunter-jenkinsx-serverless-pr-35 --type=json -p='[{"op": "add", "path": "/metadata/labels/istio-injection", "value": "enabled"}]' - kubectl patch ns jx-production --type=json -p='[{"op": "add", "path": "/metadata/labels/istio-injection", "value": "enabled"}]' + kubectl label namespace jx-staging istio-injection=enabled + kubectl label namespace jx-production istio-injection=enabled Optional: Create a `ServiceEntry` to allow traffic to the Google metadata api to display the region From eb2b1639f11370ae814e6233926f826eb7fceaab Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 30 Jan 2019 11:15:51 +0100 Subject: [PATCH 2/6] Shorter intervals for demo --- flagger/croc-hunter-canary.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flagger/croc-hunter-canary.yaml b/flagger/croc-hunter-canary.yaml index b02a8fa..b63f44a 100644 --- a/flagger/croc-hunter-canary.yaml +++ b/flagger/croc-hunter-canary.yaml @@ -26,10 +26,11 @@ spec: - public-gateway.istio-system.svc.cluster.local # Istio virtual service host names (optional) hosts: - - croc-hunter.istio.g.csanchez.org + - croc-hunter.istio.us.g.csanchez.org + - croc-hunter.istio.eu.g.csanchez.org canaryAnalysis: # schedule interval (default 60s) - interval: 1m + interval: 15s # max number of failed metric checks before rollback threshold: 5 # max traffic percentage routed to canary @@ -43,7 +44,7 @@ spec: # minimum req success rate (non 5xx responses) # percentage (0-100) threshold: 99 - interval: 1m + interval: 30s - name: istio_request_duration_seconds_bucket # maximum req duration P99 # milliseconds From 25791ad2c115786dedec4e2d4c0ac4fba881d0fe Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 30 Jan 2019 11:31:21 +0100 Subject: [PATCH 3/6] Add paths to return delays and errors --- croc-hunter.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/croc-hunter.go b/croc-hunter.go index 50efbda..191aef1 100644 --- a/croc-hunter.go +++ b/croc-hunter.go @@ -8,6 +8,8 @@ import ( "log" "net/http" "os" + "strconv" + "time" ) var release = os.Getenv("WORKFLOW_RELEASE") @@ -104,6 +106,24 @@ func handler(w http.ResponseWriter, r *http.Request) { return } + if r.URL.Path == "/delay" { + delay, _ := strconv.Atoi(r.URL.Query().Get("wait")) + if delay <= 0 { + delay = 10 + } + time.Sleep(time.Duration(delay) * time.Second) + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "{delay: %d}", delay) + return + } + + if r.URL.Path == "/status" { + code, _ := strconv.Atoi(r.URL.Query().Get("code")) + w.WriteHeader(code) + fmt.Fprintf(w, "{code: %d}", code) + return + } + hostname, err := os.Hostname() if err != nil { log.Fatalf("could not get hostname: %s", err) From 374efbb61f56a43b877fbcaa8ea909469dcc78f0 Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Wed, 30 Jan 2019 11:51:19 +0100 Subject: [PATCH 4/6] Injecting Delays and Errors --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index eb50b03..f502bc9 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,13 @@ For those that have dreamt to hunt crocs Basic go webserver to demonstrate example CI/CD pipeline using Kubernetes +## Injecting Delays and Errors + +Making requests to these urls will cause the app to delay the response or to respond with an error, and can be useful to simulate real life errors. + + /delay?wait=5 + /status?code=500 + # Deploy using JenkinsX (Kubernetes, Helm, Monocular, ChartMuseum) Just follow the [JenkinsX](http://jenkins-x.io) installation with `--prow=true` From f48798a2408b88add023aed0d1e4131b35cd5682 Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Thu, 31 Jan 2019 13:20:21 +0100 Subject: [PATCH 5/6] Add the flagger canary to the helm chart --- .../templates/canary.yaml | 53 +++++++++++++++++++ flagger/README.md | 10 ++-- 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 charts/croc-hunter-jenkinsx/templates/canary.yaml diff --git a/charts/croc-hunter-jenkinsx/templates/canary.yaml b/charts/croc-hunter-jenkinsx/templates/canary.yaml new file mode 100644 index 0000000..2f1b637 --- /dev/null +++ b/charts/croc-hunter-jenkinsx/templates/canary.yaml @@ -0,0 +1,53 @@ +{{- if eq .Release.Namespace "jx-production" }} +apiVersion: flagger.app/v1alpha2 +kind: Canary +metadata: + # canary name must match deployment name + name: jx-production-croc-hunter-jenkinsx +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: jx-production-croc-hunter-jenkinsx + # HPA reference (optional) + # autoscalerRef: + # apiVersion: autoscaling/v2beta1 + # kind: HorizontalPodAutoscaler + # name: jx-production-croc-hunter-jenkinsx + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 8080 + # Istio gateways (optional) + gateways: + - public-gateway.istio-system.svc.cluster.local + # Istio virtual service host names (optional) + hosts: + - croc-hunter.istio.us.g.csanchez.org + - croc-hunter.istio.eu.g.csanchez.org + canaryAnalysis: + # schedule interval (default 60s) + interval: 15s + # max number of failed metric checks before rollback + threshold: 5 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 10 + metrics: + - name: istio_requests_total + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 30s + - name: istio_request_duration_seconds_bucket + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s +{{- end }} diff --git a/flagger/README.md b/flagger/README.md index 63fabb1..00d534e 100644 --- a/flagger/README.md +++ b/flagger/README.md @@ -8,7 +8,7 @@ Enable Istio in the `jx-staging` and `jx-production` namespaces for metrics gath kubectl label namespace jx-production istio-injection=enabled -Create the canary object that will add our deployment to Flagger +Create the canary object that will add our deployment to Flagger. This is already created by the Helm chart when promoting to `jx-production` namespace. kubectl create -f croc-hunter-canary.yaml @@ -19,6 +19,10 @@ Optional: Create a `ServiceEntry` to allow traffic to the Google metadata api to # Grafana dashboard kubectl --namespace istio-system port-forward deploy/flagger-grafana 3000 - kubectl --namespace istio-system port-forward deploy/prometheus 9090 -[http://localhost:3000](http://localhost:3000) +Access it at [http://localhost:3000](http://localhost:3000) using admin/admin +Go to the `canary-analysis` dashboard and select + +* namespace: `jx-production` +* primary: `jx-production-croc-hunter-jenkinsx-primary` +* canary: `jx-production-croc-hunter-jenkinsx` From 8765eb82c008071f48b93a1153181971c0eec77b Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Thu, 31 Jan 2019 13:20:42 +0100 Subject: [PATCH 6/6] Import selecting go buildpack --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f502bc9..3ac9fcb 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ For example, if using GKE with cert-manager preinstalled for https certificates Then fork this repo and [import it](http://jenkins-x.io/developing/import/) - jx import --url https://github.com/GITHUB_USER/croc-hunter-jenkinsx-serverless + jx import --url https://github.com/GITHUB_USER/croc-hunter-jenkinsx-serverless --no-draft --pack=go Then, any PRs against this repo will be automatically deployed to preview environments. When they are merged they will be deployed to the `staging` environment.