From aaf314d12af1b1b2f30d25ca5489ff60ac4e23f7 Mon Sep 17 00:00:00 2001 From: Jun Zhang Date: Wed, 25 Apr 2018 11:28:43 +0800 Subject: [PATCH] build: introduce caimake to build (#36) --- .caimake/update.sh | 107 ++++++++++ Makefile | 380 ++++++++++++++++++++++++---------- build/controller/Dockerfile | 2 +- cmd/controller/app/release.go | 2 + pkg/version/base.go | 23 ++ pkg/version/version.go | 49 +++++ 6 files changed, 457 insertions(+), 106 deletions(-) create mode 100644 .caimake/update.sh create mode 100644 pkg/version/base.go create mode 100644 pkg/version/version.go diff --git a/.caimake/update.sh b/.caimake/update.sh new file mode 100644 index 00000000..9aae21a2 --- /dev/null +++ b/.caimake/update.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Exit on error. Append "|| true" if you expect an error. +set -o errexit +# Do not allow use of undefined vars. Use ${VAR:-} to use an undefined VAR +set -o nounset +# Catch the error in pipeline. +set -o pipefail + +# Controls verbosity of the script output and logging. +VERBOSE="${VERBOSE:-5}" + +# If set true, the log will be colorized +readonly COLOR_LOG=${COLOR_LOG:-true} + +readonly CAIMAKE_HOME=${CAIMAKE_HOME:-${HOME}/.caimake} + +readonly CAIMAKE_AUTO_UPDATE=${CAIMAKE_AUTO_UPDATE:-false} + +readonly project="caicloud/build-infra" + +if [[ ${COLOR_LOG} == "true" ]]; then + readonly blue="\033[34m" + readonly green="\033[32m" + readonly red="\033[31m" + readonly yellow="\033[36m" + readonly strong="\033[1m" + readonly reset="\033[0m" +else + readonly blue="" + readonly green="" + readonly red="" + readonly yellow="" + readonly strong="" + readonly reset="" +fi + +# Print a status line. Formatted to show up in a stream of output. +log::status() { + local V="${V:-0}" + if [[ $VERBOSE < $V ]]; then + return + fi + + timestamp=$(date +"[%m%d %H:%M:%S]") + echo -e "${blue}==> $timestamp${reset} ${strong}$1${reset}" + shift + for message; do + echo " $message" + done +} + +# Log an error but keep going. Don't dump the stack or exit. +log::error() { + timestamp=$(date +"[%m%d %H:%M:%S]") + echo -e "${red}!!! $timestamp${reset} ${strong}${1-}${reset}" + shift + for message; do + echo " $message" + done +} + +command_exists() { + command -v "$@" >/dev/null 2>&1 +} + +caimake::update() { + mkdir -p ${CAIMAKE_HOME} + + if ! command_exists caimake; then + # add CAIMAKE_HOME to PATH + export PATH=${CAIMAKE_HOME}:${PATH} + if ! command_exists caimake; then + # check again + caimake::download_binary + fi + fi + + if [[ ${CAIMAKE_AUTO_UPDATE-} == "true" ]]; then + caimake update + fi +} + +caimake::get_version_from_url() { + local url=${1-} + local version=${url##*/releases/download/} + version=${version%/*} + echo ${version} +} + +caimake::download_binary() { + local hostos="$(uname -s | tr '[A-Z]' '[a-z]')" + local latest="$(curl -s https://api.github.com/repos/${project}/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep -m 1 ${hostos})" + local version="$(caimake::get_version_from_url ${latest-})" + + if [[ -z ${version} ]]; then + # no release found on github + log::error "No caimake release found on Github" + exit 1 + fi + + log::status "Latest caimake release on Github is ${version}" + + curl -L ${latest} -o ${CAIMAKE_HOME}/caimake.temp + mv ${CAIMAKE_HOME}/caimake.temp ${CAIMAKE_HOME}/caimake + chmod +x ${CAIMAKE_HOME}/caimake +} diff --git a/Makefile b/Makefile index dab47f4e..de9ff8ee 100644 --- a/Makefile +++ b/Makefile @@ -1,136 +1,306 @@ # Copyright 2017 The Caicloud Authors. # -# The old school Makefile, following are required targets. The Makefile is written -# to allow building multiple binaries. You are free to add more targets or change -# existing implementations, as long as the semantics are preserved. +# ===================================================================== +# This file was autogenerated. Do not edit it manually! +# save all your custom variables and targets in Makefile.expansion! +# ===================================================================== # -# make - default to 'build' target -# make lint - code analysis -# make test - run unit test (or plus integration test) -# make build - alias to build-local target -# make build-local - build local binary targets -# make build-linux - build linux binary targets -# make container - build containers -# make push - push containers -# make clean - clean up targets +# Old-skool build tools. # -# Not included but recommended targets: -# make e2e-test -# -# The makefile is also responsible to populate project version information. -# -# TODO: implement 'make push' +# Commonly used targets (see each target for more information): +# all: Build code. +# test: Run tests. +# clean: Clean up. +# +# see https://github.com/caicloud/build-infra/blob/master/docs/Specification.md for more information. -# -# Tweak the variables based on your project. -# -# Current version of the project. -VERSION ?= v0.2.1 +# It's necessary to set this because some environments don't link sh -> bash. +SHELL := /bin/bash + +# Default target +.DEFAULT_GOAL := all + +# We don't need make's built-in rules. +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: + +# ========================================================= +# Tweak the variables based on your project. +# ========================================================= -# This repo's root import path (under GOPATH). -ROOT := github.com/caicloud/rudder +# Constants used throughout. +.EXPORT_ALL_VARIABLES: +# This controls the verbosity of the build. Higher numbers mean more output. +VERBOSE ?= 1 -# Target binaries. You can build multiple binaries for a single project. -TARGETS := controller +# If true, built on local. Otherwise, built in docker. +LOCAL_BUILD ?= true +# Golang on-build docker image. +GO_ONBUILD_IMAGE := cargo.caicloudprivatetest.com/caicloud/golang:1.9.2-alpine3.6 +# Building for these platforms. +GO_BUILD_PLATFORMS ?= linux/amd64 darwin/amd64 +# Pre-defined all directory names of targets for go build. +GO_BUILD_TARGETS := cmd/controller cmd/release-cli +# Targets using CGO_ENABLED=0. It is a single word without dir prefix. +GO_STATIC_LIBRARIES := +# Skip go unittest under the following dir. +GO_TEST_EXCEPTIONS := +# Pre-defined all directories containing Dockerfiles for building containers. +DOCKER_BUILD_TARGETS := build/controller +# Container registries. +DOCKER_REGISTRIES := cargo.caicloudprivatetest.com/caicloud +# Force pushing to override images in remote registries +DOCKER_FORCE_PUSH ?= true # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] -# $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. -IMAGE_PREFIX ?= $(strip release-) -IMAGE_SUFFIX ?= $(strip ) - -# Container registries. -REGISTRIES ?= cargo.caicloudprivatetest.com/caicloud +# $[REGISTRY] is an item from $[DOCKER_REGISTRIES], $[TARGET] is the basename from $[DOCKER_BUILD_TARGETS[@]]. +DOCKER_IMAGE_PREFIX := release- +DOCKER_IMAGE_SUFFIX := -# -# These variables should not need tweaking. -# +# ========================================================= +# variables for caimake. +# ========================================================= -# A list of all packages. -PKGS := $(shell go list ./... | grep -v /vendor | grep -v /test) +# add caimake home to path +CAIMAKE_HOME ?= $(HOME)/.caimake +PATH := $(CAIMAKE_HOME):$(PATH) +# auto update caimake binary +CAIMAKE_AUTO_UPDATE ?= true +# check if caimake.sh exists +CAIMAKE_OFFLINE := $(shell if [ -e ".caimake/caimake.sh" ]; then echo "true"; else echo "false"; fi;) +ifeq ($(CAIMAKE_OFFLINE),true) +# use local script +CAIMAKE := bash .caimake/caimake.sh +else +# use binary +CAIMAKE := caimake +endif -# Project main package location (can be multiple ones). -CMD_DIR := ./cmd +# user defined Makefile to expands targets +-include Makefile.expansion -# Project output directory. -OUTPUT_DIR := ./bin +# check if pre-build defined in Makefile.expansion +PRE_BUILD := $(shell $(MAKE) -n pre-build 2> /dev/null) +ifdef PRE_BUILD +PRE_BUILD := pre-build +endif -# Build direcotory. -BUILD_DIR := ./build +# check caimake binary +.PHONY: check-caimake +check-caimake: +ifeq ($(CAIMAKE_OFFLINE),false) + @source .caimake/update.sh && caimake::update +endif -# Git commit sha. -COMMIT := $(shell git rev-parse --short HEAD) +define ALL_HELP_INFO +# Build code. +# make all == make build +# +# Args: +# WHAT: Directory names to build. If any of these directories has a 'main' +# package, the build will produce executable files under bin/. +# If not specified, "everything" will be built. +# GOFLAGS: Extra flags to pass to 'go' when building. +# GOLDFLAGS: Extra linking flags passed to 'go' when building. +# GOGCFLAGS: Additional go compile flags passed to 'go' when building. +# +# Example: +# make +# make all or make build +# make build WHAT=cmd/server GOFLAGS=-v +# make all GOGCFLAGS="-N -l" +# Note: Use the -N -l options to disable compiler optimizations an inlining. +# Using these build options allows you to subsequently use source +# debugging tools like delve. +endef +.PHONY: all build +ifeq ($(HELP),y) +all build: + @echo "$$ALL_HELP_INFO" +else +all build: check-caimake $(PRE_BUILD) + $(CAIMAKE) go build $(WHAT) +endif -# Golang standard bin directory. -BIN_DIR := $(GOPATH)/bin -GOMETALINTER := $(BIN_DIR)/gometalinter +define GO_BUILD_HELP_INFO +# Build code. +# +# Args: +# GOFLAGS: Extra flags to pass to 'go' when building. +# GOLDFLAGS: Extra linking flags passed to 'go' when building. +# GOGCFLAGS: Additional go compile flags passed to 'go' when building. +# +# Example: +# make $(1) +# make $(1) GOFLAGS=-v +# make $(1) GOGCFLAGS="-N -l" +# Note: Use the -N -l options to disable compiler optimizations an inlining. +# Using these build options allows you to subsequently use source +# debugging tools like delve. +endef +.PHONY: $(GO_BUILD_TARGETS) +ifeq ($(HELP),y) +$(GO_BUILD_TARGETS): + $(call GO_BUILD_HELP_INFO, $@) +else +$(GO_BUILD_TARGETS): check-caimake $(PRE_BUILD) + $(CAIMAKE) go build $@ +endif +define UNITTEST_HELP_INFO +# Run uniitest # -# Define all targets. At least the following commands are required: +# Args: +# GOFLAGS: Extra flags to pass to 'go' when building. +# GOLDFLAGS: Extra linking flags passed to 'go' when building. +# GOGCFLAGS: Additional go compile flags passed to 'go' when building. # +# Example: +# make +# make unittest +# make unittest GOFLAGS="-v -x" +# make unittest GOGCFLAGS="-N -l" +# Note: Use the -N -l options to disable compiler optimizations an inlining. +# Using these build options allows you to subsequently use source +# debugging tools like delve. +endef +.PHONY: unittest +ifeq ($(HELP),y) +unittest: + @echo "$$UNITTEST_HELP_INFO" +else +unittest: check-caimake + $(CAIMAKE) go unittest +endif -# All targets. -.PHONY: lint test build container push +define BUILD_LOCAL_HELP_INFO +# Build code on local. +# +# Args: +# WHAT: Directory names to build. If any of these directories has a 'main' +# package, the build will produce executable files under bin/. +# If not specified, "everything" will be built. +# GOFLAGS: Extra flags to pass to 'go' when building. +# GOLDFLAGS: Extra linking flags passed to 'go' when building. +# GOGCFLAGS: Additional go compile flags passed to 'go' when building. +# +# Example: +# make +# make build-local +# make build-local WHAT=cmd/server GOFLAGS=-v +# make build-local GOGCFLAGS="-N -l" +# Note: Use the -N -l options to disable compiler optimizations an inlining. +# Using these build options allows you to subsequently use source +# debugging tools like delve. +endef +.PHONY: build-local +ifeq ($(HELP),y) +build-local: + @echo "$$BUILD_LOCAL_HELP_INFO" +else +build-local: check-caimake $(PRE_BUILD) + LOCAL_BUILD=true $(CAIMAKE) go build $(WHAT) +endif -build: build-local +define BUILD_IN_CONTAINER_HELP_INFO +# Build code in container. +# +# Args: +# WHAT: Directory names to build. If any of these directories has a 'main' +# package, the build will produce executable files under bin/. +# If not specified, "everything" will be built. +# GOFLAGS: Extra flags to pass to 'go' when building. +# GOLDFLAGS: Extra linking flags passed to 'go' when building. +# GOGCFLAGS: Additional go compile flags passed to 'go' when building. +# +# Example: +# make +# make build-in-container +# make build-in-container WHAT=cmd/server GOFLAGS=-v +# make build-in-container GOGCFLAGS="-N -l" +# Note: Use the -N -l options to disable compiler optimizations an inlining. +# Using these build options allows you to subsequently use source +# debugging tools like delve. +endef +.PHONY: build-in-container +ifeq ($(HELP),y) +build-in-container: + @echo "$$BUILD_LINUX_HELP_INFO" +else +build-in-container: check-caimake $(PRE_BUILD) + LOCAL_BUILD=false $(CAIMAKE) go build $(WHAT) +endif -lint: $(GOMETALINTER) - gometalinter ./... --vendor +define CONTAINER_HELP_INFO +# Build docker image. +# +# Args: +# WHAT: Directories containing Dockerfile. +# +# Example: +# make container +# make container WAHT=build/server +endef +.PHONY: container +ifeq ($(HELP),y) +container: + @echo "$$CONTAINER_HELP_INFO" +else +container: check-caimake + $(CAIMAKE) docker build $(WHAT) +endif -$(GOMETALINTER): - go get -u github.com/alecthomas/gometalinter - gometalinter --install &> /dev/null +define DOCKER_BUILD_HELP_INFO + # Build docker image. +# +# Example: +# make $(1) +endef +.PHONY: $(DOCKER_BUILD_TARGETS) +ifeq ($(HELP),y) +$(DOCKER_BUILD_TARGETS): + $(call DOCKER_BUILD_HELP_INFO, $@) +else +$(DOCKER_BUILD_TARGETS): check-caimake + $(CAIMAKE) docker build $@ +endif -test: - go test $(PKGS) +define PUSH_HELP_INFO +# Push docker image. +# You should run make container before push +# +# Args: +# WHAT: Directory names containing Dockerfile. +# +# Example: +# make push +# make push WAHT=build/server +endef +.PHONY: push +ifeq ($(HELP),y) +push: + @echo "$$PUSH_HELP_INFO" +else +push: check-caimake + $(CAIMAKE) docker push $(WHAT) +endif -build-local: - @for target in $(TARGETS); do \ - go build -i -v -o $(OUTPUT_DIR)/$${target} \ - -ldflags "-s -w -X $(ROOT)/pkg/version.VERSION=$(VERSION) \ - -X $(ROOT)/pkg/version.COMMIT=$(COMMIT) \ - -X $(ROOT)/pkg/version.REPOROOT=$(ROOT)" \ - $(CMD_DIR)/$${target}; \ - done - -build-linux: - @for target in $(TARGETS); do \ - for registry in $(REGISTRIES); do \ - docker run --rm \ - -v $(PWD):/go/src/$(ROOT) \ - -w /go/src/$(ROOT) \ - -e GOOS=linux \ - -e GOARCH=amd64 \ - -e GOPATH=/go \ - -e CGO_ENABLED=0 \ - $${registry}/golang:1.9.2-debian-jessie \ - go build -i -v -o $(OUTPUT_DIR)/$${target} \ - -ldflags "-s -w -X $(ROOT)/pkg/version.VERSION=$(VERSION) \ - -X $(ROOT)/pkg/version.COMMIT=$(COMMIT) \ - -X $(ROOT)/pkg/version.REPOROOT=$(ROOT)" \ - $(CMD_DIR)/$${target}; \ - done \ - done - -container: build-linux - @for target in $(TARGETS); do \ - for registry in $(REGISTRIES); do \ - image=$(IMAGE_PREFIX)$${target}$(IMAGE_SUFFIX); \ - docker build -t $${registry}/$${image}:$(VERSION) \ - -f $(BUILD_DIR)/$${target}/Dockerfile .; \ - done \ - done - -push: container - @for target in $(TARGETS); do \ - for registry in $(REGISTRIES); do \ - image=$(IMAGE_PREFIX)$${target}$(IMAGE_SUFFIX); \ - docker push $${registry}/$${image}:$(VERSION); \ - done \ - done +define CLEAN_HELP_INFO +# Remove all build artifacts. +# +# Example: +# make clean +# +endef .PHONY: clean +ifeq ($(HELP),y) clean: - -rm -vrf ${OUTPUT_DIR} + @echo "$$CLEAN_HELP_INFO" +else +clean: check-caimake + $(CAIMAKE) clean +endif diff --git a/build/controller/Dockerfile b/build/controller/Dockerfile index c6bfb572..5f9c5da3 100644 --- a/build/controller/Dockerfile +++ b/build/controller/Dockerfile @@ -8,7 +8,7 @@ LABEL maintainer="Wei Guo " RUN mkdir /data -COPY bin/controller /release +COPY bin/linux_amd64/controller /release ENTRYPOINT ["/release"] CMD ["-v4"] diff --git a/cmd/controller/app/release.go b/cmd/controller/app/release.go index 1de279e9..9af60a0a 100644 --- a/cmd/controller/app/release.go +++ b/cmd/controller/app/release.go @@ -8,6 +8,7 @@ import ( "github.com/caicloud/rudder/cmd/controller/app/options" "github.com/caicloud/rudder/pkg/kube" "github.com/caicloud/rudder/pkg/store" + "github.com/caicloud/rudder/pkg/version" "github.com/golang/glog" apps "k8s.io/api/apps/v1" @@ -48,6 +49,7 @@ type ControllerContext struct { // Run runs the ReleaseServer. This should never exit. func Run(s *options.ReleaseServer) error { glog.Infof("Initialize release server") + glog.Infof("Rudder Build Information, %v", version.Get().Pretty()) kubeConfig, err := clientcmd.BuildConfigFromFlags("", s.Kubeconfig) if err != nil { return err diff --git a/pkg/version/base.go b/pkg/version/base.go new file mode 100644 index 00000000..6142467d --- /dev/null +++ b/pkg/version/base.go @@ -0,0 +1,23 @@ +package version + +var ( + // semantic version, derived by build scripts (see + // https://github.com/kubernetes/community/blob/master/contributors/design-proposals/release/versioning.md + // for a detailed discussion of this field) + // + // TODO: This field is still called "gitVersion" for legacy + // reasons. For prerelease versions, the build metadata on the + // semantic version is a git hash, but the version itself is no + // longer the direct output of "git describe", but a slight + // translation to be semver compliant. + + // NOTE: The $Format strings are replaced during 'git archive' thanks to the + // companion .gitattributes file containing 'export-subst' in this same + // directory. See also https://git-scm.com/docs/gitattributes + version = "v0.0.0-master+$Format:%h$" + gitRemote = "" // git origin remote url + gitCommit = "$Format:%H$" // sha1 from git, output of $(git rev-parse HEAD) + gitTreeState = "" // state of git tree, either "clean" or "dirty" + + buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') +) diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 00000000..b7f2c002 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,49 @@ +package version + +import ( + "encoding/json" + "fmt" + "runtime" +) + +// Info contains versioning information. +// TODO: Add []string of api versions supported? It's still unclear +// how we'll want to distribute that information. +type Info struct { + Version string `json:"version"` + GitRemote string `json:"gitRemote"` + GitCommit string `json:"gitCommit"` + GitTreeState string `json:"gitTreeState"` + BuildDate string `json:"buildDate"` + GoVersion string `json:"goVersion"` + Compiler string `json:"compiler"` + Platform string `json:"platform"` +} + +// Pretty returns a pretty output representation of Info +func (info Info) Pretty() string { + str, _ := json.MarshalIndent(info, "", " ") + return string(str) +} + +func (info Info) String() string { + str, _ := json.Marshal(info) + return string(str) +} + +// Get returns the overall codebase version. It's for detecting +// what code a binary was built from. +func Get() Info { + // These variables typically come from -ldflags settings and in + // their absence fallback to the settings in pkg/version/base.go + return Info{ + Version: version, + GitRemote: gitRemote, + GitCommit: gitCommit, + GitTreeState: gitTreeState, + BuildDate: buildDate, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + } +}