diff --git a/.bazelrc b/.bazelrc
new file mode 100644
index 0000000..e16f644
--- /dev/null
+++ b/.bazelrc
@@ -0,0 +1,25 @@
+startup --expand_configs_in_place
+
+# Show us more details
+build --show_timestamps --verbose_failures
+test --test_output=errors --test_verbose_timeout_warnings
+
+# Include git version info
+build --workspace_status_command hack/print-workspace-status.sh
+
+# Preset definitions
+build --define DOCKER_REGISTRY=index.docker.io/nghialv2607
+
+# https://github.com/bazelbuild/rules_go/blob/master/go/modes.rst
+build --features=pure
+
+# Make /tmp hermetic
+build --sandbox_tmpfs_path=/tmp
+
+# Ensure that Bazel never runs as root, which can cause unit tests to fail.
+# This flag requires Bazel 0.5.0+
+build --sandbox_fake_username
+
+# Enable go race detection
+build:unit --features=race
+test:unit --features=race
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b8b40d9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,27 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+.DS_Store
+
+# Bazel
+/bazel-bin
+/bazel-genfiles
+/bazel-lotus
+/bazel-out
+/bazel-testlogs
+
+# golang
+/vendor
+
+# libsonnet
+/libsonnet/.tmp
diff --git a/BUILD.bazel b/BUILD.bazel
new file mode 100644
index 0000000..d0b5ad4
--- /dev/null
+++ b/BUILD.bazel
@@ -0,0 +1,11 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
+# gazelle:exclude vendor
+# gazelle:exclude install
+# gazelle:exclude hack
+# gazelle:build_file_name BUILD.bazel
+# gazelle:prefix github.com/nghialv/lotus
+
+gazelle(
+    name = "gazelle",
+)
diff --git a/Gopkg.lock b/Gopkg.lock
new file mode 100644
index 0000000..4d02b8e
--- /dev/null
+++ b/Gopkg.lock
@@ -0,0 +1,714 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "cloud.google.com/go"
+  packages = [
+    "compute/metadata",
+    "iam",
+    "internal",
+    "internal/optional",
+    "internal/trace",
+    "internal/version",
+    "storage"
+  ]
+  revision = "debcad1964693daf8ef4bc06292d7e828e075130"
+  version = "v0.31.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/beorn7/perks"
+  packages = ["quantile"]
+  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+  version = "v1.1.1"
+
+[[projects]]
+  name = "github.com/ghodss/yaml"
+  packages = ["."]
+  revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/gogo/protobuf"
+  packages = [
+    "proto",
+    "sortkeys"
+  ]
+  revision = "636bf0302bc95575d69441b25a2603156ffdddf1"
+  version = "v1.1.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/glog"
+  packages = ["."]
+  revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/groupcache"
+  packages = ["lru"]
+  revision = "c65c006176ff7ff98bb916961c7abbc6b0afc0aa"
+
+[[projects]]
+  name = "github.com/golang/protobuf"
+  packages = [
+    "jsonpb",
+    "proto",
+    "protoc-gen-go/descriptor",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/struct",
+    "ptypes/timestamp"
+  ]
+  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
+  version = "v1.2.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/google/btree"
+  packages = ["."]
+  revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/google/gofuzz"
+  packages = ["."]
+  revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
+
+[[projects]]
+  name = "github.com/googleapis/gax-go"
+  packages = ["."]
+  revision = "b001040cd31805261cbd978842099e326dfa857b"
+  version = "v2.0.2"
+
+[[projects]]
+  name = "github.com/googleapis/gnostic"
+  packages = [
+    "OpenAPIv2",
+    "compiler",
+    "extensions"
+  ]
+  revision = "7c663266750e7d82587642f65e60bc4083f1f84e"
+  version = "v0.2.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/gregjones/httpcache"
+  packages = [
+    ".",
+    "diskcache"
+  ]
+  revision = "9cad4c3443a7200dd6400aef47183728de563a38"
+
+[[projects]]
+  name = "github.com/hashicorp/golang-lru"
+  packages = [
+    ".",
+    "simplelru"
+  ]
+  revision = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768"
+  version = "v0.5.0"
+
+[[projects]]
+  name = "github.com/imdario/mergo"
+  packages = ["."]
+  revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
+  version = "v0.3.6"
+
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  name = "github.com/json-iterator/go"
+  packages = ["."]
+  revision = "1624edc4454b8682399def8740d46db5e4362ba4"
+  version = "v1.1.5"
+
+[[projects]]
+  name = "github.com/matttproud/golang_protobuf_extensions"
+  packages = ["pbutil"]
+  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
+  version = "v1.0.1"
+
+[[projects]]
+  name = "github.com/modern-go/concurrent"
+  packages = ["."]
+  revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
+  version = "1.0.3"
+
+[[projects]]
+  name = "github.com/modern-go/reflect2"
+  packages = ["."]
+  revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
+  version = "1.0.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/petar/GoLLRB"
+  packages = ["llrb"]
+  revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
+
+[[projects]]
+  name = "github.com/peterbourgon/diskv"
+  packages = ["."]
+  revision = "5f041e8faa004a95c88a202771f4cc3e991971e6"
+  version = "v2.0.1"
+
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/prometheus/client_golang"
+  packages = [
+    "api",
+    "api/prometheus/v1",
+    "prometheus",
+    "prometheus/internal",
+    "prometheus/promhttp"
+  ]
+  revision = "abad2d1bd44235a26707c172eab6bca5bf2dbad3"
+  version = "v0.9.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/prometheus/client_model"
+  packages = ["go"]
+  revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/prometheus/common"
+  packages = [
+    "expfmt",
+    "internal/bitbucket.org/ww/goautoneg",
+    "model"
+  ]
+  revision = "7e9e6cabbd393fc208072eedef99188d0ce788b6"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/prometheus/procfs"
+  packages = [
+    ".",
+    "internal/util",
+    "nfs",
+    "xfs"
+  ]
+  revision = "185b4288413d2a0dd0806f78c90dde719829e5ae"
+
+[[projects]]
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "298182f68c66c05229eb03ac171abe6e309ee79a"
+  version = "v1.0.3"
+
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = [
+    "assert",
+    "require"
+  ]
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
+
+[[projects]]
+  name = "go.opencensus.io"
+  packages = [
+    ".",
+    "exemplar",
+    "exporter/prometheus",
+    "internal",
+    "internal/tagencoding",
+    "plugin/ochttp",
+    "plugin/ochttp/propagation/b3",
+    "stats",
+    "stats/internal",
+    "stats/view",
+    "tag",
+    "trace",
+    "trace/internal",
+    "trace/propagation",
+    "trace/tracestate"
+  ]
+  revision = "b7bf3cdb64150a8c8c53b769fdeb2ba581bd4d4b"
+  version = "v0.18.0"
+
+[[projects]]
+  name = "go.uber.org/atomic"
+  packages = ["."]
+  revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289"
+  version = "v1.3.2"
+
+[[projects]]
+  name = "go.uber.org/multierr"
+  packages = ["."]
+  revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "go.uber.org/zap"
+  packages = [
+    ".",
+    "buffer",
+    "internal/bufferpool",
+    "internal/color",
+    "internal/exit",
+    "zapcore"
+  ]
+  revision = "ff33455a0e382e8a81d14dd7c922020b6b5e7982"
+  version = "v1.9.1"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["ssh/terminal"]
+  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "context/ctxhttp",
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "internal/timeseries",
+    "trace"
+  ]
+  revision = "c44066c5c816ec500d459a2a324a753f78531ae0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/oauth2"
+  packages = [
+    ".",
+    "google",
+    "internal",
+    "jws",
+    "jwt"
+  ]
+  revision = "8527f56f71077909d6ead7facfe18fbf05ebdf83"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sync"
+  packages = ["errgroup"]
+  revision = "42b317875d0fa942474b76e1b46a6060d720ae6e"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows"
+  ]
+  revision = "c8e336422fdcf1a7abeb865a23da98be0d8e2bc7"
+
+[[projects]]
+  name = "golang.org/x/text"
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable"
+  ]
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/time"
+  packages = ["rate"]
+  revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/tools"
+  packages = [
+    "go/ast/astutil",
+    "imports",
+    "internal/fastwalk",
+    "internal/gopathwalk"
+  ]
+  revision = "a0a13e073c7bae39af55369bcd1c2dc7ebb88ede"
+
+[[projects]]
+  branch = "master"
+  name = "google.golang.org/api"
+  packages = [
+    "gensupport",
+    "googleapi",
+    "googleapi/internal/uritemplates",
+    "googleapi/transport",
+    "internal",
+    "iterator",
+    "option",
+    "storage/v1",
+    "transport/http",
+    "transport/http/internal/propagation"
+  ]
+  revision = "0a71a4356c3f4bcbdd16294c78ca2a31fda36cca"
+
+[[projects]]
+  name = "google.golang.org/appengine"
+  packages = [
+    ".",
+    "internal",
+    "internal/app_identity",
+    "internal/base",
+    "internal/datastore",
+    "internal/log",
+    "internal/modules",
+    "internal/remote_api",
+    "internal/urlfetch",
+    "urlfetch"
+  ]
+  revision = "ae0ab99deb4dc413a2b4bd6c8bdd0eb67f1e4d06"
+  version = "v1.2.0"
+
+[[projects]]
+  branch = "master"
+  name = "google.golang.org/genproto"
+  packages = [
+    "googleapis/api/annotations",
+    "googleapis/iam/v1",
+    "googleapis/rpc/code",
+    "googleapis/rpc/status"
+  ]
+  revision = "c830210a61dfaa790e1920f8d0470fc27bc2efbe"
+
+[[projects]]
+  name = "google.golang.org/grpc"
+  packages = [
+    ".",
+    "balancer",
+    "balancer/base",
+    "balancer/roundrobin",
+    "codes",
+    "connectivity",
+    "credentials",
+    "encoding",
+    "encoding/proto",
+    "grpclog",
+    "internal",
+    "internal/backoff",
+    "internal/channelz",
+    "internal/envconfig",
+    "internal/grpcrand",
+    "internal/transport",
+    "keepalive",
+    "metadata",
+    "naming",
+    "peer",
+    "resolver",
+    "resolver/dns",
+    "resolver/passthrough",
+    "stats",
+    "status",
+    "tap"
+  ]
+  revision = "2e463a05d100327ca47ac218281906921038fd95"
+  version = "v1.16.0"
+
+[[projects]]
+  name = "gopkg.in/inf.v0"
+  packages = ["."]
+  revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
+  version = "v0.9.1"
+
+[[projects]]
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+  version = "v2.2.1"
+
+[[projects]]
+  name = "k8s.io/api"
+  packages = [
+    "admissionregistration/v1alpha1",
+    "admissionregistration/v1beta1",
+    "apps/v1",
+    "apps/v1beta1",
+    "apps/v1beta2",
+    "authentication/v1",
+    "authentication/v1beta1",
+    "authorization/v1",
+    "authorization/v1beta1",
+    "autoscaling/v1",
+    "autoscaling/v2beta1",
+    "batch/v1",
+    "batch/v1beta1",
+    "batch/v2alpha1",
+    "certificates/v1beta1",
+    "core/v1",
+    "events/v1beta1",
+    "extensions/v1beta1",
+    "networking/v1",
+    "policy/v1beta1",
+    "rbac/v1",
+    "rbac/v1alpha1",
+    "rbac/v1beta1",
+    "scheduling/v1alpha1",
+    "scheduling/v1beta1",
+    "settings/v1alpha1",
+    "storage/v1",
+    "storage/v1alpha1",
+    "storage/v1beta1"
+  ]
+  revision = "37c5ce6f2f592fbbd798bb86a8814d0918b3abe1"
+  version = "kubernetes-1.11.4"
+
+[[projects]]
+  branch = "release-1.11"
+  name = "k8s.io/apimachinery"
+  packages = [
+    "pkg/api/errors",
+    "pkg/api/meta",
+    "pkg/api/resource",
+    "pkg/apis/meta/internalversion",
+    "pkg/apis/meta/v1",
+    "pkg/apis/meta/v1/unstructured",
+    "pkg/apis/meta/v1beta1",
+    "pkg/conversion",
+    "pkg/conversion/queryparams",
+    "pkg/fields",
+    "pkg/labels",
+    "pkg/runtime",
+    "pkg/runtime/schema",
+    "pkg/runtime/serializer",
+    "pkg/runtime/serializer/json",
+    "pkg/runtime/serializer/protobuf",
+    "pkg/runtime/serializer/recognizer",
+    "pkg/runtime/serializer/streaming",
+    "pkg/runtime/serializer/versioning",
+    "pkg/selection",
+    "pkg/types",
+    "pkg/util/cache",
+    "pkg/util/clock",
+    "pkg/util/diff",
+    "pkg/util/errors",
+    "pkg/util/framer",
+    "pkg/util/intstr",
+    "pkg/util/json",
+    "pkg/util/mergepatch",
+    "pkg/util/net",
+    "pkg/util/runtime",
+    "pkg/util/sets",
+    "pkg/util/strategicpatch",
+    "pkg/util/validation",
+    "pkg/util/validation/field",
+    "pkg/util/wait",
+    "pkg/util/yaml",
+    "pkg/version",
+    "pkg/watch",
+    "third_party/forked/golang/json",
+    "third_party/forked/golang/reflect"
+  ]
+  revision = "8ee1a638bafa4ae9691077e690cb45dd54f45111"
+
+[[projects]]
+  name = "k8s.io/client-go"
+  packages = [
+    "discovery",
+    "discovery/fake",
+    "informers",
+    "informers/admissionregistration",
+    "informers/admissionregistration/v1alpha1",
+    "informers/admissionregistration/v1beta1",
+    "informers/apps",
+    "informers/apps/v1",
+    "informers/apps/v1beta1",
+    "informers/apps/v1beta2",
+    "informers/autoscaling",
+    "informers/autoscaling/v1",
+    "informers/autoscaling/v2beta1",
+    "informers/batch",
+    "informers/batch/v1",
+    "informers/batch/v1beta1",
+    "informers/batch/v2alpha1",
+    "informers/certificates",
+    "informers/certificates/v1beta1",
+    "informers/core",
+    "informers/core/v1",
+    "informers/events",
+    "informers/events/v1beta1",
+    "informers/extensions",
+    "informers/extensions/v1beta1",
+    "informers/internalinterfaces",
+    "informers/networking",
+    "informers/networking/v1",
+    "informers/policy",
+    "informers/policy/v1beta1",
+    "informers/rbac",
+    "informers/rbac/v1",
+    "informers/rbac/v1alpha1",
+    "informers/rbac/v1beta1",
+    "informers/scheduling",
+    "informers/scheduling/v1alpha1",
+    "informers/scheduling/v1beta1",
+    "informers/settings",
+    "informers/settings/v1alpha1",
+    "informers/storage",
+    "informers/storage/v1",
+    "informers/storage/v1alpha1",
+    "informers/storage/v1beta1",
+    "kubernetes",
+    "kubernetes/scheme",
+    "kubernetes/typed/admissionregistration/v1alpha1",
+    "kubernetes/typed/admissionregistration/v1beta1",
+    "kubernetes/typed/apps/v1",
+    "kubernetes/typed/apps/v1beta1",
+    "kubernetes/typed/apps/v1beta2",
+    "kubernetes/typed/authentication/v1",
+    "kubernetes/typed/authentication/v1beta1",
+    "kubernetes/typed/authorization/v1",
+    "kubernetes/typed/authorization/v1beta1",
+    "kubernetes/typed/autoscaling/v1",
+    "kubernetes/typed/autoscaling/v2beta1",
+    "kubernetes/typed/batch/v1",
+    "kubernetes/typed/batch/v1beta1",
+    "kubernetes/typed/batch/v2alpha1",
+    "kubernetes/typed/certificates/v1beta1",
+    "kubernetes/typed/core/v1",
+    "kubernetes/typed/events/v1beta1",
+    "kubernetes/typed/extensions/v1beta1",
+    "kubernetes/typed/networking/v1",
+    "kubernetes/typed/policy/v1beta1",
+    "kubernetes/typed/rbac/v1",
+    "kubernetes/typed/rbac/v1alpha1",
+    "kubernetes/typed/rbac/v1beta1",
+    "kubernetes/typed/scheduling/v1alpha1",
+    "kubernetes/typed/scheduling/v1beta1",
+    "kubernetes/typed/settings/v1alpha1",
+    "kubernetes/typed/storage/v1",
+    "kubernetes/typed/storage/v1alpha1",
+    "kubernetes/typed/storage/v1beta1",
+    "listers/admissionregistration/v1alpha1",
+    "listers/admissionregistration/v1beta1",
+    "listers/apps/v1",
+    "listers/apps/v1beta1",
+    "listers/apps/v1beta2",
+    "listers/autoscaling/v1",
+    "listers/autoscaling/v2beta1",
+    "listers/batch/v1",
+    "listers/batch/v1beta1",
+    "listers/batch/v2alpha1",
+    "listers/certificates/v1beta1",
+    "listers/core/v1",
+    "listers/events/v1beta1",
+    "listers/extensions/v1beta1",
+    "listers/networking/v1",
+    "listers/policy/v1beta1",
+    "listers/rbac/v1",
+    "listers/rbac/v1alpha1",
+    "listers/rbac/v1beta1",
+    "listers/scheduling/v1alpha1",
+    "listers/scheduling/v1beta1",
+    "listers/settings/v1alpha1",
+    "listers/storage/v1",
+    "listers/storage/v1alpha1",
+    "listers/storage/v1beta1",
+    "pkg/apis/clientauthentication",
+    "pkg/apis/clientauthentication/v1alpha1",
+    "pkg/apis/clientauthentication/v1beta1",
+    "pkg/version",
+    "plugin/pkg/client/auth/exec",
+    "plugin/pkg/client/auth/gcp",
+    "rest",
+    "rest/watch",
+    "testing",
+    "third_party/forked/golang/template",
+    "tools/auth",
+    "tools/cache",
+    "tools/clientcmd",
+    "tools/clientcmd/api",
+    "tools/clientcmd/api/latest",
+    "tools/clientcmd/api/v1",
+    "tools/metrics",
+    "tools/pager",
+    "tools/record",
+    "tools/reference",
+    "transport",
+    "util/buffer",
+    "util/cert",
+    "util/connrotation",
+    "util/flowcontrol",
+    "util/homedir",
+    "util/integer",
+    "util/jsonpath",
+    "util/retry",
+    "util/workqueue"
+  ]
+  revision = "3db8bfc8858dc9a5d6e7ef5817f58a7ca30b0c6a"
+  version = "kubernetes-1.11.4"
+
+[[projects]]
+  branch = "release-1.11"
+  name = "k8s.io/code-generator"
+  packages = [
+    "cmd/client-gen",
+    "cmd/client-gen/args",
+    "cmd/client-gen/generators",
+    "cmd/client-gen/generators/fake",
+    "cmd/client-gen/generators/scheme",
+    "cmd/client-gen/generators/util",
+    "cmd/client-gen/path",
+    "cmd/client-gen/types",
+    "pkg/util"
+  ]
+  revision = "8c97d6ab64da020f8b151e9d3ed8af3172f5c390"
+
+[[projects]]
+  branch = "master"
+  name = "k8s.io/gengo"
+  packages = [
+    "args",
+    "generator",
+    "namer",
+    "parser",
+    "types"
+  ]
+  revision = "7338e4bfd6915369a1375890db1bbda0158c9863"
+
+[[projects]]
+  branch = "master"
+  name = "k8s.io/kube-openapi"
+  packages = ["pkg/util/proto"]
+  revision = "0d1aeffe1c68f49accbd05c185ae534fe1372a3f"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "afbd1e4e846fd871664c2620d98badea04974915c613d86e143ec755b1b92d8b"
+  solver-name = "gps-cdcl"
+  solver-version = 1
diff --git a/Gopkg.toml b/Gopkg.toml
new file mode 100644
index 0000000..4f5d466
--- /dev/null
+++ b/Gopkg.toml
@@ -0,0 +1,57 @@
+required = [
+  "k8s.io/code-generator/cmd/client-gen"
+]
+
+[[constraint]]
+  name = "go.uber.org/zap"
+  version = "1.9.1"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "=v0.0.3"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.2.2"
+
+[[constraint]]
+  name = "go.opencensus.io"
+  version = "=v0.18.0"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "v0.9.1"
+
+[[constraint]]
+  name = "k8s.io/client-go"
+  version = "kubernetes-1.11.4"
+
+[[constraint]]
+  name = "k8s.io/apimachinery"
+  version = "kubernetes-1.11.4"
+
+[[constraint]]
+  name = "k8s.io/api"
+  version = "kubernetes-1.11.4"
+
+[[constraint]]
+  name = "k8s.io/code-generator"
+  version = "kubernetes-1.11.4"
+
+[prune]
+  non-go = true
+  go-tests = true
+  unused-packages = true
+
+  [[prune.project]]
+    name = "k8s.io/code-generator"
+    unused-packages = false
+    non-go = false
+    go-tests = false
+
+  [[prune.project]]
+    name = "k8s.io/gengo"
+    unused-packages = false
+    non-go = false
+    go-tests = false
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..70a29fc
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Le Van Nghia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..2525d2a
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,45 @@
+.PHONY: build
+build:
+	bazel build -k -- //cmd/... //pkg/...
+
+.PHONY: test
+test:
+	bazel test -- //cmd/... //pkg/...
+
+.PHONY: push
+push:
+	bazel run --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //cmd:push_all_images
+
+.PHONY: dep
+dep:
+	dep ensure
+	bazel run //:gazelle -- update-repos -from_file=Gopkg.lock
+
+.PHONY: gazelle
+gazelle:
+	bazel run //:gazelle
+
+.PHONY: codegen
+codegen:
+	./hack/update-codegen.sh
+
+.PHONY: libsonnet
+libsonnet:
+	jb update --jsonnetpkg-home=libsonnet
+
+.PHONY: install
+install:
+	helm install --name lotus -f ./install/values.yaml ./install/helm
+
+.PHONY: upgrade
+upgrade:
+	helm upgrade lotus -f ./install/values.yaml ./install/helm
+
+.PHONY: generate-manifests
+generate-manifests:
+	./hack/generate-manifests.sh
+	./hack/generate-manifests.sh norbac
+
+.PHONY: generate-dashboards
+generate-dashboards:
+	./hack/generate-dashboards.sh
diff --git a/NOTICE.md b/NOTICE.md
new file mode 100644
index 0000000..202b69c
--- /dev/null
+++ b/NOTICE.md
@@ -0,0 +1,10 @@
+In this project, I used and modified the following files from the Kubernetes project.
+So those files are containing the LICENSE from Kubernetes project.
+
+Lotus:
+- https://github.com/nghialv/lotus/blob/master/hack/print-workspace-status.sh
+- https://github.com/nghialv/lotus/blob/master/pkg/version/def.bzl
+
+Kubernetes:
+- https://github.com/kubernetes/kubernetes/blob/master/hack/print-workspace-status.sh
+- https://github.com/kubernetes/kubernetes/blob/master/pkg/version/def.bzl
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..adbac51
--- /dev/null
+++ b/README.md
@@ -0,0 +1,161 @@
+# Lotus [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat)](http://makeapullrequest.com) [![MIT Licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/nghialv/lotus/blob/master/LICENSE)
+
+Lotus is a Kubernetes controller for running load testing. Lotus schedules & monitors the load test workers, collects & stores the metrics and notifies the test result.
+
+Once installed, Lotus provides the following features:
+- GRPC and HTTP support
+- Ability to write the scenario by any language you want
+- Automation-friendly
+  - `Checks` (like asserts, fails in normal test) for easy and flexible CI configuration
+  - Test is configured by using declarative Kubernetes CRD for version control friendliness
+- Flexible metrics storage and visualization
+  - Ability to view visualized time series by Grafana
+  - Ability to persist time series data to a long-term storage like GCS or S3
+  - Ability to notify and store the result summary to multiple receivers: GCS, Slack, Logger...
+
+> _I am thinking about adding a feature that helps us determine the maximum number of  users (requests) the target services can handle. This can be done by automatically running the load tests with the number of virtual users increasing gradually until one of the checks fails. Or a feature that helps us determine the needed resources of the target services so that they can handle the given number of users. [more](https://github.com/nghialv/lotus/issues/1)_
+
+### Installation
+Firstly, you need to install Lotus controller on your Kubernetes cluster to start using.
+Lotus requires a Kubernetes cluster of version `>=1.9.0`.
+
+The Lotus controller can be installed either by using the helm [`chart`](https://github.com/nghialv/lotus/tree/master/install/helm) or by using Kubernetes [`manifests`](https://github.com/nghialv/lotus/tree/master/install/manifests) directly.
+(Using the helm chart is recommended.)
+
+``` console
+helm install --name lotus ./install/helm
+```
+
+See [`install`](https://github.com/nghialv/lotus/tree/master/install) for more details.
+
+### Running Lotus
+We have 2 steps to start running a load test:
+- Writing a load test scenario
+- Writing a Lotus CRD configuration
+
+#### 1. Writing a load test scenario
+
+Theoretically, you can write your scenarios by using any language you like. The only thing you need to have is a metrics exporter for Prometheus.
+
+In the case of Golang, I have already prepared some util packages (e.g. [`metrics`](https://github.com/nghialv/lotus/tree/master/pkg/metrics), [`virtualuser`](https://github.com/nghialv/lotus/tree/master/pkg/virtualuser)) that help you write your scenarios faster and easier.
+
+- Expose a metrics server in your scenario's `main.go`
+``` go
+import "github.com/nghialv/lotus/pkg/metrics"
+
+m, err := metrics.NewServer(8081)
+if err != nil {
+    return err
+}
+defer m.Stop()
+go m.Run()
+```
+- In case you want to send gRPC's rpcs to your load server, let's set `grpcmetrics.ClientHandler` as the `StatsHandler` of your gRPC connection.
+``` go
+grpc.Dial(
+    grpc.WithStatsHandler(&grpcmetrics.ClientHandler{}),
+)
+```
+
+- In case you want to send HTTP requests to your load server, let's use the `Transport` from `httpmetrics` package.
+``` go
+http.Client{
+    Transport: &httpmetrics.Transport{},
+}
+```
+- That is all. Now let's build your scenario image and publish to your container registry.
+
+#### 2. Writing a Lotus CRD configuration
+
+``` yaml
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: simple-scenario-12345                                  // The unique testID
+spec:
+  worker:
+    runTime: 10m                                               // How long the load test will be run
+    replicas: 15                                               // How many workers should be created
+    metricsPort: 8081                                          // What port number should be used to collect metrics
+    containers:
+      - name: worker
+        image: your-registry/your-worker-image                 // The scenario image you published above
+        ports:
+          - name: metrics
+            containerPort: 8081
+  checks:                                                      // You can add some checks to be checked while running
+    - name: GRPCHighErrorRate
+      expr: lotus_grpc_client_failure_percentage > 10
+      for: 30s
+```
+
+Then apply this file to your Kubernetes cluster. Lotus will handle this test for you.
+
+See [`crd-configurations.md`](https://github.com/nghialv/lotus/blob/master/docs/lotus-crd-configurations.md) for all configurable fields.
+
+See [`examples`](https://github.com/nghialv/lotus/tree/master/examples) for more examples.
+
+### Outputs
+
+- Test summary
+
+Lotus collects the metrics data and evaluates the `checks` to build a summary result for each test.
+Lotus can be configured to upload this summary file to external services (e.g: GCS, Slack...) or to log into `stdout`.
+3 formats of the summary file are supported: `Text`, `Markdown`, `JSON`.
+
+``` yaml
+TestID:        test-scenario-12345
+TestStatus:    Succeeded
+Start:         09:02:59 2018-12-03
+End:           09:12:59 2018-12-03
+
+MetricsSummary:
+
+1. Virtual User
+  - Started:             1M
+  - Failed:              0
+
+2. GRPC
+  - RPCTotal:            25M
+  - FailurePercentage:   2.507
+
+GroupByMethod:
+                        RPCs      Failure%  Latency   SentBytes  RecvBytes
+
+  - helloworld.Hello    12.5M     1.015     105       15        8
+  - helloworld.Profile  12.5M     1.415     152       8         256
+  - all                 25M       1.207     135       12        245
+
+Grafana: http://localhost:3000/dashboard/db/grpc?from=1543827779598&to=1543828379598
+```
+
+
+- Grafana dashboards
+
+To be able to fully explore and understand your test, Lotus is providing some Grafana dashboards to view the visualizations of the metrics.
+You can also set up Lotus to persist the time series data to a long-term storage (GCS or S3) for accessing after the test is deleted.
+
+- Test Status
+
+After applying the Lotus CRD to your Kubernetes cluster you can also use the following command to check the status of your test.
+
+``` console
+kubectl describe Lotus your-lotus-name
+```
+
+Your test can be one of these status: `Pending`, `Preparing`, `Running`, `Cleaning`, `FailureCleaning`, `Failed`, `Succeeded`
+
+### Examples
+
+Please checkout [`/examples`](https://github.com/nghialv/lotus/tree/master/examples) directory that contains some prepared examples.
+
+### FQA
+
+Refer to [FQA.md](https://github.com/nghialv/lotus/blob/master/docs/fqa.md)
+
+### Development
+
+Refer to [development.md](https://github.com/nghialv/lotus/blob/master/docs/development.md)
+
+### LICENSE
+Lotus is released under the MIT license. See [LICENSE](https://github.com/nghialv/lotus/blob/master/LICENSE) file for the details.
diff --git a/WORKSPACE b/WORKSPACE
new file mode 100644
index 0000000..3a4a34d
--- /dev/null
+++ b/WORKSPACE
@@ -0,0 +1,394 @@
+workspace(
+    name = "lotus",
+)
+
+load(
+    "@bazel_tools//tools/build_defs/repo:http.bzl",
+    "http_archive",
+)
+
+# Rules go
+http_archive(
+    name = "io_bazel_rules_go",
+    urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.16.2/rules_go-0.16.2.tar.gz"],
+    sha256 = "f87fa87475ea107b3c69196f39c82b7bbf58fe27c62a338684c20ca17d1d8613",
+)
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_rules_dependencies",
+    "go_register_toolchains",
+)
+
+go_rules_dependencies()
+
+go_register_toolchains(
+    go_version = "1.11.2",
+)
+
+# Gazelle
+http_archive(
+    name = "bazel_gazelle",
+    urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.15.0/bazel-gazelle-0.15.0.tar.gz"],
+    sha256 = "6e875ab4b6bf64a38c352887760f21203ab054676d9c1b274963907e0768740d",
+)
+
+load(
+    "@bazel_gazelle//:deps.bzl",
+    "gazelle_dependencies",
+    "go_repository",
+)
+
+gazelle_dependencies()
+
+# Docker
+http_archive(
+    name = "io_bazel_rules_docker",
+    sha256 = "29d109605e0d6f9c892584f07275b8c9260803bf0c6fcb7de2623b2bedc910bd",
+    strip_prefix = "rules_docker-0.5.1",
+    urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.5.1.tar.gz"],
+)
+
+load(
+    "@io_bazel_rules_docker//go:image.bzl",
+    _go_image_repos = "repositories",
+)
+
+_go_image_repos()
+
+# Protoc-gen-validate
+go_repository(
+    name = "com_lyft_protoc_gen_validate",
+    tag = "v0.0.11",
+    importpath = "github.com/lyft/protoc-gen-validate",
+    #build_file_proto_mode = "disable",
+)
+
+# Below is the list autogenerated go_repository.
+
+go_repository(
+    name = "com_github_davecgh_go_spew",
+    commit = "8991bc29aa16c548c550c7ff78260e27b9ab7c73",
+    importpath = "github.com/davecgh/go-spew",
+)
+
+go_repository(
+    name = "com_github_pmezard_go_difflib",
+    commit = "792786c7400a136282c1664665ae0a8db921c6c2",
+    importpath = "github.com/pmezard/go-difflib",
+)
+
+go_repository(
+    name = "com_github_stretchr_testify",
+    commit = "f35b8ab0b5a2cef36673838d662e249dd9c94686",
+    importpath = "github.com/stretchr/testify",
+)
+
+go_repository(
+    name = "org_uber_go_atomic",
+    commit = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289",
+    importpath = "go.uber.org/atomic",
+)
+
+go_repository(
+    name = "org_uber_go_multierr",
+    commit = "3c4937480c32f4c13a875a1829af76c98ca3d40a",
+    importpath = "go.uber.org/multierr",
+)
+
+go_repository(
+    name = "org_uber_go_zap",
+    commit = "ff33455a0e382e8a81d14dd7c922020b6b5e7982",
+    importpath = "go.uber.org/zap",
+)
+
+go_repository(
+    name = "com_github_ghodss_yaml",
+    commit = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7",
+    importpath = "github.com/ghodss/yaml",
+)
+
+go_repository(
+    name = "com_github_gogo_protobuf",
+    commit = "636bf0302bc95575d69441b25a2603156ffdddf1",
+    importpath = "github.com/gogo/protobuf",
+)
+
+go_repository(
+    name = "com_github_golang_glog",
+    commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998",
+    importpath = "github.com/golang/glog",
+)
+
+go_repository(
+    name = "com_github_golang_protobuf",
+    commit = "aa810b61a9c79d51363740d207bb46cf8e620ed5",
+    importpath = "github.com/golang/protobuf",
+)
+
+go_repository(
+    name = "com_github_google_btree",
+    commit = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306",
+    importpath = "github.com/google/btree",
+)
+
+go_repository(
+    name = "com_github_google_gofuzz",
+    commit = "24818f796faf91cd76ec7bddd72458fbced7a6c1",
+    importpath = "github.com/google/gofuzz",
+)
+
+go_repository(
+    name = "com_github_googleapis_gnostic",
+    build_file_proto_mode = "disable",
+    commit = "7c663266750e7d82587642f65e60bc4083f1f84e",
+    importpath = "github.com/googleapis/gnostic",
+)
+
+go_repository(
+    name = "com_github_gregjones_httpcache",
+    commit = "9cad4c3443a7200dd6400aef47183728de563a38",
+    importpath = "github.com/gregjones/httpcache",
+)
+
+go_repository(
+    name = "com_github_imdario_mergo",
+    commit = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4",
+    importpath = "github.com/imdario/mergo",
+)
+
+go_repository(
+    name = "com_github_json_iterator_go",
+    commit = "1624edc4454b8682399def8740d46db5e4362ba4",
+    importpath = "github.com/json-iterator/go",
+)
+
+go_repository(
+    name = "com_github_modern_go_concurrent",
+    commit = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94",
+    importpath = "github.com/modern-go/concurrent",
+)
+
+go_repository(
+    name = "com_github_modern_go_reflect2",
+    commit = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd",
+    importpath = "github.com/modern-go/reflect2",
+)
+
+go_repository(
+    name = "com_github_petar_gollrb",
+    commit = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4",
+    importpath = "github.com/petar/GoLLRB",
+)
+
+go_repository(
+    name = "com_github_peterbourgon_diskv",
+    commit = "5f041e8faa004a95c88a202771f4cc3e991971e6",
+    importpath = "github.com/peterbourgon/diskv",
+)
+
+go_repository(
+    name = "com_github_spf13_pflag",
+    commit = "298182f68c66c05229eb03ac171abe6e309ee79a",
+    importpath = "github.com/spf13/pflag",
+)
+
+go_repository(
+    name = "in_gopkg_inf_v0",
+    commit = "d2d2541c53f18d2a059457998ce2876cc8e67cbf",
+    importpath = "gopkg.in/inf.v0",
+)
+
+go_repository(
+    name = "in_gopkg_yaml_v2",
+    commit = "5420a8b6744d3b0345ab293f6fcba19c978f1183",
+    importpath = "gopkg.in/yaml.v2",
+)
+
+go_repository(
+    name = "io_k8s_api",
+    build_file_proto_mode = "disable",
+    commit = "37c5ce6f2f592fbbd798bb86a8814d0918b3abe1",
+    importpath = "k8s.io/api",
+)
+
+go_repository(
+    name = "io_k8s_apimachinery",
+    build_file_proto_mode = "disable",
+    commit = "8ee1a638bafa4ae9691077e690cb45dd54f45111",
+    importpath = "k8s.io/apimachinery",
+)
+
+go_repository(
+    name = "io_k8s_client_go",
+    commit = "3db8bfc8858dc9a5d6e7ef5817f58a7ca30b0c6a",
+    importpath = "k8s.io/client-go",
+)
+
+go_repository(
+    name = "org_golang_google_appengine",
+    commit = "ae0ab99deb4dc413a2b4bd6c8bdd0eb67f1e4d06",
+    importpath = "google.golang.org/appengine",
+)
+
+go_repository(
+    name = "org_golang_x_crypto",
+    commit = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06",
+    importpath = "golang.org/x/crypto",
+)
+
+go_repository(
+    name = "org_golang_x_net",
+    commit = "c44066c5c816ec500d459a2a324a753f78531ae0",
+    importpath = "golang.org/x/net",
+)
+
+go_repository(
+    name = "org_golang_x_oauth2",
+    commit = "8527f56f71077909d6ead7facfe18fbf05ebdf83",
+    importpath = "golang.org/x/oauth2",
+)
+
+go_repository(
+    name = "org_golang_x_sys",
+    commit = "c8e336422fdcf1a7abeb865a23da98be0d8e2bc7",
+    importpath = "golang.org/x/sys",
+)
+
+go_repository(
+    name = "org_golang_x_text",
+    commit = "f21a4dfb5e38f5895301dc265a8def02365cc3d0",
+    importpath = "golang.org/x/text",
+)
+
+go_repository(
+    name = "org_golang_x_time",
+    commit = "fbb02b2291d28baffd63558aa44b4b56f178d650",
+    importpath = "golang.org/x/time",
+)
+
+go_repository(
+    name = "io_k8s_code_generator",
+    commit = "8c97d6ab64da020f8b151e9d3ed8af3172f5c390",
+    importpath = "k8s.io/code-generator",
+)
+
+go_repository(
+    name = "io_k8s_gengo",
+    commit = "7338e4bfd6915369a1375890db1bbda0158c9863",
+    importpath = "k8s.io/gengo",
+)
+
+go_repository(
+    name = "org_golang_x_tools",
+    commit = "a0a13e073c7bae39af55369bcd1c2dc7ebb88ede",
+    importpath = "golang.org/x/tools",
+)
+
+go_repository(
+    name = "com_github_hashicorp_golang_lru",
+    commit = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768",
+    importpath = "github.com/hashicorp/golang-lru",
+)
+
+go_repository(
+    name = "io_k8s_kube_openapi",
+    commit = "0d1aeffe1c68f49accbd05c185ae534fe1372a3f",
+    importpath = "k8s.io/kube-openapi",
+)
+
+go_repository(
+    name = "com_github_golang_groupcache",
+    commit = "c65c006176ff7ff98bb916961c7abbc6b0afc0aa",
+    importpath = "github.com/golang/groupcache",
+)
+
+go_repository(
+    name = "com_google_cloud_go",
+    commit = "debcad1964693daf8ef4bc06292d7e828e075130",
+    importpath = "cloud.google.com/go",
+)
+
+go_repository(
+    name = "com_github_beorn7_perks",
+    commit = "3a771d992973f24aa725d07868b467d1ddfceafb",
+    importpath = "github.com/beorn7/perks",
+)
+
+go_repository(
+    name = "com_github_matttproud_golang_protobuf_extensions",
+    commit = "c12348ce28de40eed0136aa2b644d0ee0650e56c",
+    importpath = "github.com/matttproud/golang_protobuf_extensions",
+)
+
+go_repository(
+    name = "com_github_prometheus_client_golang",
+    commit = "abad2d1bd44235a26707c172eab6bca5bf2dbad3",
+    importpath = "github.com/prometheus/client_golang",
+)
+
+go_repository(
+    name = "com_github_prometheus_client_model",
+    commit = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f",
+    importpath = "github.com/prometheus/client_model",
+)
+
+go_repository(
+    name = "com_github_prometheus_common",
+    commit = "7e9e6cabbd393fc208072eedef99188d0ce788b6",
+    importpath = "github.com/prometheus/common",
+)
+
+go_repository(
+    name = "com_github_prometheus_procfs",
+    commit = "185b4288413d2a0dd0806f78c90dde719829e5ae",
+    importpath = "github.com/prometheus/procfs",
+)
+
+go_repository(
+    name = "io_opencensus_go",
+    commit = "b7bf3cdb64150a8c8c53b769fdeb2ba581bd4d4b",
+    importpath = "go.opencensus.io",
+)
+
+go_repository(
+    name = "org_golang_google_genproto",
+    commit = "c830210a61dfaa790e1920f8d0470fc27bc2efbe",
+    importpath = "google.golang.org/genproto",
+)
+
+go_repository(
+    name = "org_golang_google_grpc",
+    commit = "2e463a05d100327ca47ac218281906921038fd95",
+    importpath = "google.golang.org/grpc",
+)
+
+go_repository(
+    name = "org_golang_x_sync",
+    commit = "42b317875d0fa942474b76e1b46a6060d720ae6e",
+    importpath = "golang.org/x/sync",
+)
+
+go_repository(
+    name = "com_github_googleapis_gax_go",
+    commit = "b001040cd31805261cbd978842099e326dfa857b",
+    importpath = "github.com/googleapis/gax-go",
+)
+
+go_repository(
+    name = "org_golang_google_api",
+    commit = "0a71a4356c3f4bcbdd16294c78ca2a31fda36cca",
+    importpath = "google.golang.org/api",
+)
+
+go_repository(
+    name = "com_github_inconshreveable_mousetrap",
+    commit = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75",
+    importpath = "github.com/inconshreveable/mousetrap",
+)
+
+go_repository(
+    name = "com_github_spf13_cobra",
+    commit = "ef82de70bb3f60c65fb8eebacbb2d122ef517385",
+    importpath = "github.com/spf13/cobra",
+)
diff --git a/cmd/BUILD.bazel b/cmd/BUILD.bazel
new file mode 100644
index 0000000..e5ee396
--- /dev/null
+++ b/cmd/BUILD.bazel
@@ -0,0 +1,15 @@
+load(":image.bzl", "all_images")
+load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
+
+container_bundle(
+    name = "bundle_to_push",
+    images = all_images(),
+    stamp = True,
+)
+
+load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
+
+docker_push(
+    name = "push_all_images",
+    bundle = ":bundle_to_push",
+)
diff --git a/cmd/example/BUILD.bazel b/cmd/example/BUILD.bazel
new file mode 100644
index 0000000..9f9a079
--- /dev/null
+++ b/cmd/example/BUILD.bazel
@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/nghialv/lotus/cmd/example",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/app/example/cmd/helloworld:go_default_library",
+        "//pkg/app/example/cmd/simplegrpc:go_default_library",
+        "//pkg/app/example/cmd/simplehttp:go_default_library",
+        "//pkg/app/example/cmd/threesteps:go_default_library",
+        "//pkg/app/example/cmd/virtualuser:go_default_library",
+        "//pkg/cli:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "example",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+
+go_image(
+    name = "image",
+    binary = ":example",
+    visibility = ["//visibility:public"],
+)
diff --git a/cmd/example/main.go b/cmd/example/main.go
new file mode 100644
index 0000000..5306af6
--- /dev/null
+++ b/cmd/example/main.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+	"log"
+
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/app/example/cmd/helloworld"
+	"github.com/nghialv/lotus/pkg/app/example/cmd/simplegrpc"
+	"github.com/nghialv/lotus/pkg/app/example/cmd/simplehttp"
+	"github.com/nghialv/lotus/pkg/app/example/cmd/threesteps"
+	"github.com/nghialv/lotus/pkg/app/example/cmd/virtualuser"
+)
+
+func main() {
+	app := cli.NewApp(
+		"lotus-example",
+		"Example of using lotus.",
+	)
+	app.AddCommands(
+		simplehttp.NewCommand(),
+		simplegrpc.NewCommand(),
+		threesteps.NewCommand(),
+		virtualuser.NewCommand(),
+		helloworld.NewCommand(),
+	)
+	if err := app.Run(); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/cmd/image.bzl b/cmd/image.bzl
new file mode 100644
index 0000000..bb5a922
--- /dev/null
+++ b/cmd/image.bzl
@@ -0,0 +1,12 @@
+def all_images():
+    cmds = {
+		"lotus": "lotus",
+		"example": "lotus-example",
+    }
+    images = {}
+
+    for cmd, repo in cmds.items():
+        images["$(DOCKER_REGISTRY)/%s:{STABLE_GIT_COMMIT_FULL}" % repo] = "//cmd/%s:image" % cmd
+        images["$(DOCKER_REGISTRY)/%s:{STABLE_GIT_COMMIT}" % repo ] = "//cmd/%s:image" % cmd
+
+    return images
diff --git a/cmd/lotus/BUILD.bazel b/cmd/lotus/BUILD.bazel
new file mode 100644
index 0000000..e794b0e
--- /dev/null
+++ b/cmd/lotus/BUILD.bazel
@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/nghialv/lotus/cmd/lotus",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/app/lotus/cmd/controller:go_default_library",
+        "//pkg/app/lotus/cmd/monitor:go_default_library",
+        "//pkg/cli:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "lotus",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+
+go_image(
+    name = "image",
+    binary = ":lotus",
+    visibility = ["//visibility:public"],
+)
diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go
new file mode 100644
index 0000000..934bd14
--- /dev/null
+++ b/cmd/lotus/main.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"log"
+
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/app/lotus/cmd/controller"
+	"github.com/nghialv/lotus/pkg/app/lotus/cmd/monitor"
+)
+
+func main() {
+	app := cli.NewApp(
+		"lotus",
+		"Load testing tool.",
+	)
+	app.AddCommands(
+		controller.NewCommand(),
+		monitor.NewCommand(),
+	)
+	if err := app.Run(); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..2641d5d
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,7 @@
+# Documentations
+
+- [Life of a Lotus CRD](https://github.com/nghialv/lotus/blob/master/docs/life-of-lotus-crd.md)
+- [Controller's configurations](https://github.com/nghialv/lotus/blob/master/docs/configurations.md)
+- [Lotus CRD's configurations](https://github.com/nghialv/lotus/blob/master/docs/lotus-crd-configurations.md)
+- [Frequently Questioned Answers](https://github.com/nghialv/lotus/blob/master/docs/fqa.md)
+- [How to contribute](https://github.com/nghialv/lotus/blob/master/docs/development.md)
diff --git a/docs/configurations.md b/docs/configurations.md
new file mode 100644
index 0000000..8620e57
--- /dev/null
+++ b/docs/configurations.md
@@ -0,0 +1,84 @@
+# Configurations
+
+This is an example of a full configuration file.
+
+``` yaml
+lotus:
+  configs:
+    checks:                                             // 1. The list of global checks. Those checks will be applied to all tests.
+      - name: NoWorker
+        expr: absent(up)
+        for: 30s
+      - name: HasWorkerDown
+        expr: up == 0
+        for: 30s
+      - name: GRPCHighFailurePercentage
+        expr: lotus_grpc_client_completed_rpcs_failure_percentage:method > 5
+        for: 30s
+      - name: HTTPHighFailurePercentage
+        expr: lotus_http_client_completed_requests_5xx_percentage:host:route:method > 5
+        for: 30s
+      - name: VirtualUserHighFailurePercentage
+        expr: lotus_virtual_user_failure_percentage > 2
+        for: 10s
+    receivers:                                          // 2. The list of all receivers to send the summary result.
+      - name: gcs
+        gcs:
+          bucket: lotus-result-bucket
+          credentials:
+            secret: gcs-credentials
+            file: gcs-credentials.json
+      - name: lotus-slack-channel
+        slack:
+          hookUrl: https://hooks.slack.com/services/YOUR-HOOK
+      - name: logger
+        logger:
+    timeSeriesStorage:                                  // 3. A long-term storage for storing time series data.
+      gcs:
+        bucket: lotus-timeseries-bucket
+        credentials:
+          secret: gcs-credentials
+          file: gcs-credentials.json
+    grafanaBaseUrl: http://your-grafana-domain:3000
+```
+
+### 1. Global checks setup
+
+You can define the checks on your Lotus CRD for each test, but you can also define some global checks on the configuration file.
+I recommend to add these 2 checks to your global checks: the first one is for checking if no worker has started, the second one is for checking if has any worker down.
+
+```
+lotus:
+  configs:
+    checks:
+      - name: NoWorker
+        expr: absent(up)
+        for: 30s
+      - name: HasWorkerDown
+        expr: up == 0
+        for: 30s
+```
+
+### 2. Receivers setup
+
+Currently we are supporting 3 types of receiver: GCS, Slack, Logger.
+
+#### GCS
+
+To configure Google Cloud Storage as a receiver, you need to set receiver with GCS bucket name and k8s secret that contains the Google Application credentials.
+
+```
+receivers:
+  - name: gcs
+    gcs:
+      bucket: lotus-result-bucket       // The bucket name created on Google Cloud Storage
+      credentials:
+        secret: gcs-credentials         // The name of k8s secret that contains Google Application credentials 
+        file: gcs-credentials.json      // The credentials file name inside the secret
+```
+
+
+### 3. Long term storage setup
+
+To able to access the time series data after your test is deleted you have to configure to store those time series data to a long-term storage like GCS, S3, Azure...
+You can do that by adding configuration for `lotus.configs.timeSEriesStorage` field.
diff --git a/docs/development.md b/docs/development.md
new file mode 100644
index 0000000..3e07c49
--- /dev/null
+++ b/docs/development.md
@@ -0,0 +1,55 @@
+# Development
+
+### Prerequisites
+
+- [bazel](https://github.com/bazelbuild/bazel) (>= `0.17.1`)
+- [jsonnet](https://jsonnet.org/) (Only if you want to make a change for the grafana dashboards.)
+
+### Getting started
+
+- Building
+``` console
+make build
+```
+
+- Testing
+``` console
+make test
+```
+
+- Adding a new go dependency
+``` console
+### 1. Update Gopkg.toml file
+
+### 2. Fetch the dependency and update Gopkg.lock by running
+make dep
+
+### 3. Update bazel's BUILD files by running
+make gazelle
+```
+
+- Making a change on [`Lotus model`](https://github.com/nghialv/lotus/blob/master/pkg/app/lotus/apis/lotus/v1beta1/types.go)
+
+We are using [`code-generator`](https://github.com/kubernetes/code-generator) to generate a typed client, informers, listers and deep-copy functions for `Lotus model`.
+Then after making a change on the `Lotus model` you have to run the following command to update the generated codes.
+``` console
+make codegen
+```
+The following files and directories will be updated.
+
+```
+pkg/lotus/apis/lotus/v1beta1/zz_generated.deepcopy.go
+pkg/lotus/client/
+```
+
+- Making a change on grafana dashboards
+
+We are using `jsonnet` to do dashboard templating. The templates is located at [/install/dashboard-templates](https://github.com/nghialv/lotus/tree/master/install/dashboard-templates)
+
+``` console
+### Regenerate grafana dashoards
+make generate-dashboards
+
+### Regenerate kubernetes manifests with the new updates
+make generate-manifests
+```
diff --git a/docs/fqa.md b/docs/fqa.md
new file mode 100644
index 0000000..f1e5287
--- /dev/null
+++ b/docs/fqa.md
@@ -0,0 +1,13 @@
+# Frequently Questioned Answers
+
+- Can we store the timeseries data of the metrics to a long-term storage?
+
+  Yes.
+
+- What storage will be supported?
+
+  GCS and S3 (near future)
+
+- What does `3m` mean in the summary?
+
+https://en.wikipedia.org/wiki/Metric_prefix
diff --git a/docs/life-of-lotus-crd.md b/docs/life-of-lotus-crd.md
new file mode 100644
index 0000000..03d49e6
--- /dev/null
+++ b/docs/life-of-lotus-crd.md
@@ -0,0 +1,3 @@
+# Life of a Lotus CRD
+
+![](https://github.com/nghialv/lotus/blob/master/docs/life-of-lotus-crd.png)
diff --git a/docs/life-of-lotus-crd.png b/docs/life-of-lotus-crd.png
new file mode 100644
index 0000000..096e5af
Binary files /dev/null and b/docs/life-of-lotus-crd.png differ
diff --git a/docs/lotus-crd-configurations.md b/docs/lotus-crd-configurations.md
new file mode 100644
index 0000000..d01ac9e
--- /dev/null
+++ b/docs/lotus-crd-configurations.md
@@ -0,0 +1,57 @@
+# Lotus CRD Configurations
+
+The following is an example of the full configurations.
+
+``` yaml
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: scenario-12345
+spec:
+  ttlSecondsAfterFinished: 300
+  checkIntervalSeconds: 10
+  checkInitialDelaySeconds: 15
+  worker:
+    runTime: 30m
+    replicas: 20
+    metricsPort: 8081
+    containers:
+      - name: worker
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=worker
+          - --helloworld-grpc-address=helloworld:8080
+          - --helloworld-http-address=http://helloworld:9090
+        ports:
+          - name: metrics
+            containerPort: 8081
+        volumeMounts:
+          - name: data
+            mountPath: /etc/data
+    volumes:
+      - name: data
+        configMap:
+          name: worker-data
+  preparer:
+    containers:
+      - name: preparer
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=preparer
+  cleaner:
+    containers:
+      - name: cleaner
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=cleaner
+  checks:
+    - name: GRPCHighLatency
+      expr: lotus_grpc_client_roundtrip_latency:method > 250
+      for: 30s
+    - name: VirtualUserHighFailurePercentage
+      expr: lotus_virtual_user_failure_percentage > 10
+      for: 10s
+```
\ No newline at end of file
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 0000000..942bfb7
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,39 @@
+# Examples
+
+### Prerequisites
+
+Before running one of these examples, you have to deploy [`helloworld`](https://github.com/nghialv/lotus/tree/master/examples/helloworld) target service to your cluster by running the following command.
+``` console
+kubectl apply -f /helloworld
+```
+The implementation of `helloworld` server is here [helloworld.go](https://github.com/nghialv/lotus/blob/master/pkg/app/example/cmd/helloworld/helloworld.go)
+
+When running this service will start one grpc server for handling rpcs defined in [helloworld.proto](https://github.com/nghialv/lotus/blob/master/pkg/app/example/helloworld/helloworld.proto) and one HTTP server for handling incoming HTTP requests.
+
+### simple-http-scenario
+
+- Scenario: [`pkg/app/example/cmd/simplehttp/scenario.go`](https://github.com/nghialv/lotus/blob/master/pkg/app/example/cmd/simplehttp/scenario.go)
+- Lotus CRD: [`simple-http-scenario.yaml`](https://github.com/nghialv/lotus/blob/master/examples/simple-http-scenario.yaml)
+
+A simple scenario that send one http request to `http://httpbin.org/`.
+
+### simple-grpc-scenario
+
+- Scenario: [`/pkg/app/example/cmd/simplegrpc/scenario.go`](https://github.com/nghialv/lotus/blob/master/pkg/app/example/cmd/simplegrpc/scenario.go)
+- Lotus CRD: [`simple-grpc-scenario.yaml`](https://github.com/nghialv/lotus/blob/master/examples/simple-grpc-scenario.yaml)
+
+A simple scenario that send one grpc request to `helloworld` service.
+
+### three-steps-scenario
+
+- Scenario: [`/pkg/app/example/cmd/threesteps/scenario.go`](https://github.com/nghialv/lotus/blob/master/pkg/app/example/cmd/threesteps/scenario.go)
+- Lotus CRD: [`three-steps-scenario.yaml`](https://github.com/nghialv/lotus/blob/master/examples/three-steps-scenario.yaml)
+
+An example containing full 3 steps of Lotus: `preparer`, `worker` and `cleaner`.
+
+### virtual-user-scenario
+
+- Scenario: [`/pkg/app/example/cmd/virtualuser/scenario.go`](https://github.com/nghialv/lotus/blob/master/pkg/app/example/cmd/virtualuser/scenario.go)
+- Lotus CRD: [`virtualuser-scenario.yaml`](https://github.com/nghialv/lotus/blob/master/examples/virtualuser-scenario.yaml)
+
+An example using [`virtualuser`](https://github.com/nghialv/lotus/tree/master/pkg/virtualuser) package to spawn a given number of virtual users on each worker.
diff --git a/examples/helloworld/deployment.yaml b/examples/helloworld/deployment.yaml
new file mode 100644
index 0000000..96beb2d
--- /dev/null
+++ b/examples/helloworld/deployment.yaml
@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: helloworld
+  labels:
+    app: helloworld
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: helloworld
+  template:
+    metadata:
+      labels:
+        app: helloworld
+    spec:
+      containers:
+      - name: helloworld
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - helloworld
+        ports:
+        - name: grpc
+          containerPort: 8080
+        - name: http
+          containerPort: 9090
diff --git a/examples/helloworld/service.yaml b/examples/helloworld/service.yaml
new file mode 100644
index 0000000..0fdd4fe
--- /dev/null
+++ b/examples/helloworld/service.yaml
@@ -0,0 +1,14 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: helloworld
+spec:
+  selector:
+    app: helloworld
+  ports:
+  - name: grpc
+    protocol: TCP
+    port: 8080
+  - name: http
+    protocol: TCP
+    port: 9090
diff --git a/examples/simple-grpc-scenario.yaml b/examples/simple-grpc-scenario.yaml
new file mode 100644
index 0000000..c832f3c
--- /dev/null
+++ b/examples/simple-grpc-scenario.yaml
@@ -0,0 +1,18 @@
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: simple-grpc-scenario-123456789
+spec:
+  worker:
+    runTime: 3m
+    replicas: 2
+    metricsPort: 8081
+    containers:
+      - name: worker
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - simple-grpc-scenario
+          - --helloworld-grpc-address=helloworld:8080
+        ports:
+          - name: metrics
+            containerPort: 8081
diff --git a/examples/simple-http-scenario.yaml b/examples/simple-http-scenario.yaml
new file mode 100644
index 0000000..5d2c78e
--- /dev/null
+++ b/examples/simple-http-scenario.yaml
@@ -0,0 +1,17 @@
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: simple-http-scenario-123456789
+spec:
+  worker:
+    runTime: 3m
+    replicas: 2
+    metricsPort: 8081
+    containers:
+      - name: worker
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - simple-http-scenario
+        ports:
+          - name: metrics
+            containerPort: 8081
diff --git a/examples/three-steps-scenario.yaml b/examples/three-steps-scenario.yaml
new file mode 100644
index 0000000..8c8e5ae
--- /dev/null
+++ b/examples/three-steps-scenario.yaml
@@ -0,0 +1,45 @@
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: three-steps-scenario-1
+spec:
+  checkIntervalSeconds: 10
+  worker:
+    runTime: 3m
+    replicas: 2
+    metricsPort: 8081
+    containers:
+      - name: worker
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=worker
+          - --helloworld-grpc-address=helloworld:8080
+          - --helloworld-http-address=http://helloworld:9090
+        ports:
+          - name: metrics
+            containerPort: 8081
+  preparer:
+    containers:
+      - name: preparer
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=preparer
+          - --duration=10s
+          - --helloworld-grpc-address=helloworld:8080
+          - --helloworld-http-address=http://helloworld:9090
+  cleaner:
+    containers:
+      - name: cleaner
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - three-steps-scenario
+          - --step=cleaner
+          - --duration=10s
+          - --helloworld-grpc-address=helloworld:8080
+          - --helloworld-http-address=http://helloworld:9090
+  checks:
+    - name: GRPCHighLatency
+      expr: lotus_grpc_client_roundtrip_latency:method > 2500
+      for: 30s
diff --git a/examples/virtualuser-scenario.yaml b/examples/virtualuser-scenario.yaml
new file mode 100644
index 0000000..b75815d
--- /dev/null
+++ b/examples/virtualuser-scenario.yaml
@@ -0,0 +1,25 @@
+apiVersion: lotus.nghialv.com/v1beta1
+kind: Lotus
+metadata:
+  name: virtual-user-scenario-12345
+spec:
+  checkIntervalSeconds: 10
+  worker:
+    runTime: 3m
+    replicas: 2
+    metricsPort: 8081
+    containers:
+      - name: worker
+        image: nghialv2607/lotus-example:v0.1.0
+        args:
+          - virtual-user-scenario
+          - --num-virtual-users=100
+          - --hatch-rate=10
+          - --helloworld-grpc-address=helloworld:8080
+        ports:
+          - name: metrics
+            containerPort: 8081
+  checks:
+    - name: VirtualUserHasFailed
+      expr: lotus_virtual_user_count{virtual_user_status=~"failed"} > 0
+      for: 1s
diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt
new file mode 100644
index 0000000..7ed7ed1
--- /dev/null
+++ b/hack/boilerplate.go.txt
@@ -0,0 +1,5 @@
+/*
+
+Generated by using code-generator
+
+*/
diff --git a/hack/generate-dashboards.sh b/hack/generate-dashboards.sh
new file mode 100755
index 0000000..70790f5
--- /dev/null
+++ b/hack/generate-dashboards.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT=$(dirname ${BASH_SOURCE})/..
+LIBSONNET_DIR="$ROOT/libsonnet"
+TEMPLATES_DIR="$ROOT/install/dashboard-templates"
+DASHBOARDS_DIR="$ROOT/install/helm/dashboards"
+
+mkdir -p $DASHBOARDS_DIR
+rm -rf $DASHBOARDS_DIR/*
+
+for f in $(find $TEMPLATES_DIR -name "*-dashboard.jsonnet"); do
+  fn=${f##*/}
+  echo "Rendering $fn..."
+  jsonnet -J $LIBSONNET_DIR $f > $DASHBOARDS_DIR/${fn%.*}.json
+done
+
diff --git a/hack/generate-manifests.sh b/hack/generate-manifests.sh
new file mode 100755
index 0000000..3a02471
--- /dev/null
+++ b/hack/generate-manifests.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+OPTION=${1:-""}
+SUFFIX=""
+if [ ! -z "$OPTION" ]; then
+  SUFFIX="-$OPTION"
+fi
+
+ROOT=$(dirname ${BASH_SOURCE})/..
+MANIFESTS_DIR="${ROOT}/install/manifests${SUFFIX}"
+VALUE_FILE="${ROOT}/install/manifest-generate-values${SUFFIX}.yaml"
+HELM_CHART_DIR="${ROOT}/install/helm"
+
+echo "Generating manifests to tmp..."
+helm template --name lotus -f $VALUE_FILE $HELM_CHART_DIR --output-dir /tmp
+
+echo "Deleting all old manifests..."
+mkdir -p ${MANIFESTS_DIR}
+rm -rf ${MANIFESTS_DIR}/*
+
+echo "Copying generated manifests to manifests folder..."
+cp /tmp/lotus/templates/* ${MANIFESTS_DIR}
+for f in $(find /tmp/lotus/charts/grafana/templates -type f); do
+  cp $f ${MANIFESTS_DIR}/grafana-${f##*/};
+done
+
+echo "Deleting tmp data..."
+rm -rf /tmp/lotus
+
+echo "Done"
+
diff --git a/hack/print-workspace-status.sh b/hack/print-workspace-status.sh
new file mode 100755
index 0000000..928264c
--- /dev/null
+++ b/hack/print-workspace-status.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See https://github.com/nghialv/lotus/tree/master/NOTICE.md
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GIT_COMMIT="$(git describe --tags --always --dirty)"
+GIT_COMMIT_FULL="$(git rev-parse HEAD)"
+BUILD_DATE="$(date -u '+%Y%m%d')"
+VERSION="${BUILD_DATE}-${GIT_COMMIT}"
+
+cat <<EOF
+STABLE_GIT_COMMIT ${GIT_COMMIT}
+STABLE_GIT_COMMIT_FULL ${GIT_COMMIT_FULL}
+STABLE_BUILD_DATE ${BUILD_DATE}
+STABLE_VERSION ${VERSION}
+gitCommit ${GIT_COMMIT}
+gitCommitFull ${GIT_COMMIT_FULL}
+buildDate ${BUILD_DATE}
+version ${VERSION}
+EOF
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
new file mode 100755
index 0000000..69eb9bc
--- /dev/null
+++ b/hack/update-codegen.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)}
+
+# generate the code with:
+# --output-base    because this script should also be able to run inside the vendor dir of
+#                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
+#                  instead of the $GOPATH directly. For normal projects this can be dropped.
+${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
+  github.com/nghialv/lotus/pkg/app/lotus/client github.com/nghialv/lotus/pkg/app/lotus/apis \
+  lotus:v1beta1 \
+  --output-base "$(dirname ${BASH_SOURCE})/../../../.." \
+  --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt
diff --git a/install/README.md b/install/README.md
new file mode 100644
index 0000000..e5e7718
--- /dev/null
+++ b/install/README.md
@@ -0,0 +1,36 @@
+# Installation
+
+We are supporting 2 ways to install Lotus:
+- using helm chart
+- using kubernetes manifests directly
+
+**Using the helm chart is recommended.**
+
+### Using Helm chart
+
+The Lotus chart is put in `./helm` directory.
+
+```
+helm install --name lotus ./helm
+
+### If you want to override the values
+
+helm install --name lotus -f ./path/to/your/values.yaml ./helm
+```
+
+Please check out [`values.yaml`](https://github.com/nghialv/lotus/blob/master/install/helm/values.yaml) for configurable fields.
+Note: Please change [`grafana.adminPassword`](https://github.com/nghialv/lotus/tree/master/install/helm/values.yaml#L27) value. The current password is `admin`.
+
+### Using kubernetes manifests
+
+All kubernetes manifests for Lotus are put in `./manifests` (RBAC is enabled) and `./manifests-norbac` (RBAC is disabled) directories. We generated those manifests from the helm chart above.
+
+```
+kubectl apply -f ./manifests
+
+### Or for disabling RBAC
+
+kubectl apply -f ./manifests-norbac
+```
+
+Note: Please change [`grafana adminPassword`](https://github.com/nghialv/lotus/tree/master/install/manifests/grafana-secret.yaml#L15) value to a `base64` encoded value. The current password is `admin`.
\ No newline at end of file
diff --git a/install/dashboard-templates/common.jsonnet b/install/dashboard-templates/common.jsonnet
new file mode 100644
index 0000000..4e98b9d
--- /dev/null
+++ b/install/dashboard-templates/common.jsonnet
@@ -0,0 +1,72 @@
+local grafana = import 'grafonnet/grafana.libsonnet';
+local template = grafana.template;
+local graphPanel = grafana.graphPanel;
+local text = grafana.text;
+
+{
+  datasources:: {
+      default:: 'thanos',
+  },
+  dashboard:: {
+    schemaVersion:: 16,
+  },
+  tags:: {
+    grpc:: 'grpc',
+    http:: 'http',
+  },
+  format:: {
+    short:: 'short',
+    second:: 's',
+    millisecond:: 'ms',
+    bytes:: 'bytes',
+    bytesPerSecond:: 'Bps',
+    percent_0_100:: 'percent',
+    percent_0_1:: 'percentunit',
+  },
+  templates:: {
+    test:: template.new(
+      name='testId',
+      label='TestID',
+      datasource= $.datasources.default,
+      query='query_result(count by(job) (count_over_time(up[$__range])))',
+      regex='/"(.*)-worker"/',
+      refresh='time',
+    ),
+    hiddenCustom(
+      name,
+      value,
+    ):: template.custom(
+      name=name,
+      query=value,
+      current=value,
+      hide='value',
+    ),
+  },
+  panel:: {
+    new(
+      title,
+      format= $.format.short,
+      datasource= $.datasources.default,
+    ):: graphPanel.new(
+        title=title,
+        datasource=datasource,
+        format=format,
+        fill=2,
+        linewidth=2,
+        legend_alignAsTable=true,
+        legend_values=true,
+        legend_max=true,
+        legend_min=true,
+        legend_avg=true,
+        legend_current=true,
+        legend_sort="current",
+        legend_sortDesc=true,
+      ),
+    transparentText(
+      title=''
+    ):: text.new(
+      title=title,
+      transparent=true,
+    )
+  },
+}
diff --git a/install/dashboard-templates/grpc-dashboard.jsonnet b/install/dashboard-templates/grpc-dashboard.jsonnet
new file mode 100644
index 0000000..0cec271
--- /dev/null
+++ b/install/dashboard-templates/grpc-dashboard.jsonnet
@@ -0,0 +1,21 @@
+local grafana = import 'grafonnet/grafana.libsonnet';
+local panels = import 'panels.jsonnet';
+local common=import 'common.jsonnet';
+local dashboard = grafana.dashboard;
+local template = grafana.template;
+
+dashboard.new(
+  'GRPC',
+  tags=[common.tags.grpc],
+  time_from='now-1h',
+  schemaVersion=common.dashboard.schemaVersion,
+)
+.addTemplate(common.templates.test)
+.addPanel(panels.workerNum, { w: 12, h: 6, x: 0, y: 0 })
+.addPanel(panels.virtualUserNum, { w: 12, h: 6, x: 12, y: 0 })
+.addPanel(panels.rpcsPerSecond, { w: 12, h: 8, x: 0, y: 6 })
+.addPanel(panels.rpcLatency, { w: 12, h: 8, x: 12, y: 6 })
+.addPanel(panels.rpcsPerSecondByStatus, { w: 12, h: 8, x: 0, y: 14 })
+.addPanel(panels.percentageFailedRPCs, { w: 12, h: 8, x: 12, y: 14 })
+.addPanel(panels.rpcSentBytes, { w: 12, h: 8, x: 0, y: 22 })
+.addPanel(panels.rpcReceivedBytes, { w: 12, h: 8, x: 12, y: 22 })
\ No newline at end of file
diff --git a/install/dashboard-templates/http-dashboard.jsonnet b/install/dashboard-templates/http-dashboard.jsonnet
new file mode 100644
index 0000000..4d1505f
--- /dev/null
+++ b/install/dashboard-templates/http-dashboard.jsonnet
@@ -0,0 +1,20 @@
+local grafana = import 'grafonnet/grafana.libsonnet';
+local panels = import 'panels.jsonnet';
+local common=import 'common.jsonnet';
+local dashboard = grafana.dashboard;
+local template = grafana.template;
+
+dashboard.new(
+  'HTTP',
+  tags=[common.tags.http],
+  time_from='now-1h',
+  schemaVersion=common.dashboard.schemaVersion,
+)
+.addTemplate(common.templates.test)
+.addPanel(panels.workerNum, { w: 12, h: 6, x: 0, y: 0 })
+.addPanel(panels.virtualUserNum, { w: 12, h: 6, x: 12, y: 0 })
+.addPanel(panels.httpRequestsPerSecond, { w: 12, h: 8, x: 0, y: 6 })
+.addPanel(panels.percentageOf5xxRequests, { w: 12, h: 8, x: 12, y: 6 })
+.addPanel(panels.httpRequestLatency, { w: 12, h: 8, x: 0, y: 14 })
+.addPanel(panels.httpRequestSentBytes, { w: 12, h: 8, x: 0, y: 22 })
+.addPanel(panels.httpRequestReceivedBytes, { w: 12, h: 8, x: 12, y: 22 })
\ No newline at end of file
diff --git a/install/dashboard-templates/panels.jsonnet b/install/dashboard-templates/panels.jsonnet
new file mode 100644
index 0000000..ac45e51
--- /dev/null
+++ b/install/dashboard-templates/panels.jsonnet
@@ -0,0 +1,124 @@
+local grafana = import 'grafonnet/grafana.libsonnet';
+local common=import 'common.jsonnet';
+local prometheus = grafana.prometheus;
+local graphPanel = grafana.graphPanel;
+
+{
+  ### Worker & VirtualUser pannels
+
+  workerNum:: common.panel.new(
+    title='Number of workers',
+  )
+  .addTarget(prometheus.target(
+    'sum (up{job=~"$testId-worker"})',
+    legendFormat=' ')
+  ),
+
+  virtualUserNum:: common.panel.new(
+    title='Number of virtual users',
+  )
+  .addTarget(prometheus.target(
+    'sum by (virtual_user_status) (lotus_virtual_user_count{job=~"$testId-worker"})',
+    legendFormat='virtual_user_status')
+  ),
+
+  ### GRPC pannels
+
+  rpcsPerSecond:: common.panel.new(
+    title='RPCs / second',
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_completed_rpcs_per_second:method{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_method }}')
+  ),
+
+  rpcsPerSecondByStatus:: common.panel.new(
+    title='RPCs / seconds grouping by status',
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_completed_rpcs_per_second:status{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_status }}')
+  ),
+
+  percentageFailedRPCs:: common.panel.new(
+    title='Percentage of failed RPCs',
+    format=common.format.percent_0_100,
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_completed_rpcs_failure_percentage:method{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_method }}')
+  ),
+
+  rpcLatency:: common.panel.new(
+    title='Latency',
+    format=common.format.millisecond,
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_roundtrip_latency:method{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_method }}')
+  ),
+
+  rpcSentBytes:: common.panel.new(
+    title='Sent Bytes',
+    format=common.format.bytes,
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_sent_bytes_per_rpc:method{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_method }}')
+  ),
+
+  rpcReceivedBytes:: common.panel.new(
+    title='Received Bytes',
+    format=common.format.bytes,
+  )
+  .addTarget(prometheus.target(
+    'lotus_grpc_client_received_bytes_per_rpc:method{job=~"$testId-worker"}',
+    legendFormat='{{ grpc_client_method }}')
+  ),
+
+  ### HTTP Pannels
+
+  httpRequestsPerSecond:: common.panel.new(
+    title='Requests / second',
+  )
+  .addTarget(prometheus.target(
+    'lotus_http_client_completed_requests_per_second:host:route:method{job=~"$testId-worker"}',
+    legendFormat='{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}')
+  ),
+
+  percentageOf5xxRequests:: common.panel.new(
+    title='Percentage of 5xx Requests',
+    format=common.format.percent_0_100,
+  )
+  .addTarget(prometheus.target(
+    'lotus_http_client_completed_requests_5xx_percentage:host:route:method{job=~"$testId-worker"}',
+    legendFormat='{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}')
+  ),
+
+  httpRequestLatency:: common.panel.new(
+    title='Latency',
+    format=common.format.millisecond,
+  )
+  .addTarget(prometheus.target(
+    'lotus_http_client_roundtrip_latency:host:route:method{job=~"$testId-worker"}',
+    legendFormat='{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}')
+  ),
+
+  httpRequestSentBytes:: common.panel.new(
+    title='Sent Bytes',
+    format=common.format.bytes,
+  )
+  .addTarget(prometheus.target(
+    'lotus_http_client_sent_bytes:host:route:method{job=~"$testId-worker"}',
+    legendFormat='{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}')
+  ),
+
+  httpRequestReceivedBytes:: common.panel.new(
+    title='Received Bytes',
+    format=common.format.bytes,
+  )
+  .addTarget(prometheus.target(
+    'lotus_http_client_received_bytes:host:route:method{job=~"$testId-worker"}',
+    legendFormat='{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}')
+  ),
+}
\ No newline at end of file
diff --git a/install/helm/.helmignore b/install/helm/.helmignore
new file mode 100644
index 0000000..f0c1319
--- /dev/null
+++ b/install/helm/.helmignore
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
diff --git a/install/helm/Chart.yaml b/install/helm/Chart.yaml
new file mode 100644
index 0000000..107d77a
--- /dev/null
+++ b/install/helm/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Lotus
+name: lotus
+version: 0.1.0
diff --git a/install/helm/charts/grafana-1.19.0.tgz b/install/helm/charts/grafana-1.19.0.tgz
new file mode 100644
index 0000000..c2aff73
Binary files /dev/null and b/install/helm/charts/grafana-1.19.0.tgz differ
diff --git a/install/helm/dashboards/grpc-dashboard.json b/install/helm/dashboards/grpc-dashboard.json
new file mode 100644
index 0000000..1a898d6
--- /dev/null
+++ b/install/helm/dashboards/grpc-dashboard.json
@@ -0,0 +1,756 @@
+{
+   "annotations": {
+      "list": [ ]
+   },
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 0,
+            "y": 0
+         },
+         "id": 2,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "sum (up{job=~\"$testId-worker\"})",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": " ",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Number of workers",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 12,
+            "y": 0
+         },
+         "id": 3,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "virtual_user_status",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Number of virtual users",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 6
+         },
+         "id": 4,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_completed_rpcs_per_second:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_method }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "RPCs / second",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 6
+         },
+         "id": 5,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_roundtrip_latency:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_method }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "ms",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "ms",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 14
+         },
+         "id": 6,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_completed_rpcs_per_second:status{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_status }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "RPCs / seconds grouping by status",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 14
+         },
+         "id": 7,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_completed_rpcs_failure_percentage:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_method }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Percentage of failed RPCs",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "percent",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "percent",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 22
+         },
+         "id": 8,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_sent_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_method }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Sent Bytes",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 22
+         },
+         "id": 9,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_grpc_client_received_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ grpc_client_method }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Received Bytes",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      }
+   ],
+   "refresh": "",
+   "rows": [ ],
+   "schemaVersion": 16,
+   "style": "dark",
+   "tags": [
+      "grpc"
+   ],
+   "templating": {
+      "list": [
+         {
+            "allValue": null,
+            "current": { },
+            "datasource": "thanos",
+            "hide": 0,
+            "includeAll": false,
+            "label": "TestID",
+            "multi": false,
+            "name": "testId",
+            "options": [ ],
+            "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+            "refresh": 2,
+            "regex": "/\"(.*)-worker\"/",
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "browser",
+   "title": "GRPC",
+   "version": 0
+}
diff --git a/install/helm/dashboards/http-dashboard.json b/install/helm/dashboards/http-dashboard.json
new file mode 100644
index 0000000..300ab5c
--- /dev/null
+++ b/install/helm/dashboards/http-dashboard.json
@@ -0,0 +1,671 @@
+{
+   "annotations": {
+      "list": [ ]
+   },
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 0,
+            "y": 0
+         },
+         "id": 2,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "sum (up{job=~\"$testId-worker\"})",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": " ",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Number of workers",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 12,
+            "y": 0
+         },
+         "id": 3,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "virtual_user_status",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Number of virtual users",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 6
+         },
+         "id": 4,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_http_client_completed_requests_per_second:host:route:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Requests / second",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "short",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 6
+         },
+         "id": 5,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_http_client_completed_requests_5xx_percentage:host:route:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Percentage of 5xx Requests",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "percent",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "percent",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 14
+         },
+         "id": 6,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_http_client_roundtrip_latency:host:route:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "ms",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "ms",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 22
+         },
+         "id": 7,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_http_client_sent_bytes:host:route:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Sent Bytes",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      },
+      {
+         "aliasColors": { },
+         "bars": false,
+         "dashLength": 10,
+         "dashes": false,
+         "datasource": "thanos",
+         "fill": 2,
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 22
+         },
+         "id": 8,
+         "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "sort": "current",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+         },
+         "lines": true,
+         "linewidth": 2,
+         "links": [ ],
+         "nullPointMode": "null",
+         "percentage": false,
+         "pointradius": 5,
+         "points": false,
+         "renderer": "flot",
+         "repeat": null,
+         "seriesOverrides": [ ],
+         "spaceLength": 10,
+         "stack": false,
+         "steppedLine": false,
+         "targets": [
+            {
+               "expr": "lotus_http_client_received_bytes:host:route:method{job=~\"$testId-worker\"}",
+               "format": "time_series",
+               "intervalFactor": 2,
+               "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+               "refId": "A"
+            }
+         ],
+         "thresholds": [ ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Received Bytes",
+         "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+         },
+         "type": "graph",
+         "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": [ ]
+         },
+         "yaxes": [
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            },
+            {
+               "format": "bytes",
+               "label": null,
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true
+            }
+         ]
+      }
+   ],
+   "refresh": "",
+   "rows": [ ],
+   "schemaVersion": 16,
+   "style": "dark",
+   "tags": [
+      "http"
+   ],
+   "templating": {
+      "list": [
+         {
+            "allValue": null,
+            "current": { },
+            "datasource": "thanos",
+            "hide": 0,
+            "includeAll": false,
+            "label": "TestID",
+            "multi": false,
+            "name": "testId",
+            "options": [ ],
+            "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+            "refresh": 2,
+            "regex": "/\"(.*)-worker\"/",
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "browser",
+   "title": "HTTP",
+   "version": 0
+}
diff --git a/install/helm/requirements.lock b/install/helm/requirements.lock
new file mode 100644
index 0000000..22045ea
--- /dev/null
+++ b/install/helm/requirements.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: grafana
+  repository: https://kubernetes-charts.storage.googleapis.com
+  version: 1.19.0
+digest: sha256:b67a4b06d8fcf832a7c1e2b330d7d845c38910d2f60e3ce1e007c8761d10c710
+generated: 2018-11-30T11:08:39.379367813+09:00
diff --git a/install/helm/requirements.yaml b/install/helm/requirements.yaml
new file mode 100644
index 0000000..20961b2
--- /dev/null
+++ b/install/helm/requirements.yaml
@@ -0,0 +1,5 @@
+dependencies:
+- name: grafana
+  version: "1.19.0"
+  repository: "@stable"
+  condition: grafana.enabled
diff --git a/install/helm/templates/NOTES.txt b/install/helm/templates/NOTES.txt
new file mode 100644
index 0000000..e69de29
diff --git a/install/helm/templates/_helpers.tpl b/install/helm/templates/_helpers.tpl
new file mode 100644
index 0000000..96352d5
--- /dev/null
+++ b/install/helm/templates/_helpers.tpl
@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "lotus.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "lotus.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "lotus.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/install/helm/templates/controller-config-configmap.yaml b/install/helm/templates/controller-config-configmap.yaml
new file mode 100644
index 0000000..343e063
--- /dev/null
+++ b/install/helm/templates/controller-config-configmap.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "lotus.fullname" . }}-controller-config
+data:
+  config.yaml: |
+{{ toYaml .Values.lotus.configs | indent 4 }}
diff --git a/install/helm/templates/controller-deployment.yaml b/install/helm/templates/controller-deployment.yaml
new file mode 100644
index 0000000..00e2175
--- /dev/null
+++ b/install/helm/templates/controller-deployment.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "lotus.fullname" . }}-controller
+  labels:
+    app: {{ template "lotus.fullname" . }}-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "lotus.fullname" . }}-controller
+  template:
+    metadata:
+      labels:
+        app: {{ template "lotus.fullname" . }}-controller
+    spec:
+{{- if .Values.lotus.rbac.enabled }}
+      serviceAccountName: {{ template "lotus.fullname" . }}-controller
+{{- end }}
+      containers:
+      - name: lotus-controller
+        image: {{ .Values.lotus.image.repository }}:{{ .Values.lotus.image.tag }}
+        args:
+        - controller
+        - --log-level=debug
+        - --config-file=/etc/lotus/config.yaml
+        - --namespace={{ .Release.Namespace }}
+        - --release={{ .Release.Name }}
+{{- if .Values.lotus.rbac.enabled }}
+        - --prometheus-service-account={{ template "lotus.fullname" . }}-prometheus
+{{- end }}
+        volumeMounts:
+        - name: config
+          mountPath: /etc/lotus
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          name: {{ template "lotus.fullname" . }}-controller-config
diff --git a/install/helm/templates/controller-rbac.yaml b/install/helm/templates/controller-rbac.yaml
new file mode 100644
index 0000000..fc17d48
--- /dev/null
+++ b/install/helm/templates/controller-rbac.yaml
@@ -0,0 +1,65 @@
+{{- if .Values.lotus.rbac.enabled }}
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-controller
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-controller
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+      - apps
+      - extensions
+    resources:
+      - pods
+      - deployments
+      - statefulsets
+      - services
+      - configmaps
+      - secrets
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - "lotus.nghialv.com"
+    resources:
+      - lotuses
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-controller
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "lotus.fullname" . }}-controller
+subjects:
+- kind: ServiceAccount
+  name: {{ template "lotus.fullname" . }}-controller
+  namespace: {{ .Release.Namespace }}
+{{- end }}
\ No newline at end of file
diff --git a/install/helm/templates/crd.yaml b/install/helm/templates/crd.yaml
new file mode 100644
index 0000000..c9468dd
--- /dev/null
+++ b/install/helm/templates/crd.yaml
@@ -0,0 +1,44 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: lotuses.lotus.nghialv.com
+spec:
+  group: lotus.nghialv.com
+  version: v1beta1
+  scope: Namespaced
+  names:
+    kind: Lotus
+    plural: lotuses
+    singular: lotus
+    categories:
+      - all
+  additionalPrinterColumns:
+    - name: Phase
+      type: string
+      description: The current phase of Lotus
+      JSONPath: .status.phase
+    - name: WorkerReplicas
+      type: integer
+      description: The number of workers launched for this Lotus
+      JSONPath: .spec.worker.replicas
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          required:
+            - worker
+        status:
+          properties:
+            phase:
+              type: string
+              enum:
+              - "Pending"
+              - "Preparing"
+              - "Running"
+              - "Cleaning"
+              - "FailureCleaning"
+              - "Succeeded"
+              - "Failed"
diff --git a/install/helm/templates/grafana-dashboards-configmap.yaml b/install/helm/templates/grafana-dashboards-configmap.yaml
new file mode 100644
index 0000000..bfe1c8c
--- /dev/null
+++ b/install/helm/templates/grafana-dashboards-configmap.yaml
@@ -0,0 +1,9 @@
+{{- if .Values.grafana.enabled }}
+kind: ConfigMap
+metadata:
+  name: {{ template "lotus.fullname" . }}-grafana-dashboards
+  labels:
+    lotus-grafana-dashboard: "true"
+data:
+  {{- (.Files.Glob "dashboards/*.json").AsConfig | nindent 2 }}
+{{- end }}
\ No newline at end of file
diff --git a/install/helm/templates/grafana-datasources-configmap.yaml b/install/helm/templates/grafana-datasources-configmap.yaml
new file mode 100644
index 0000000..ad14a41
--- /dev/null
+++ b/install/helm/templates/grafana-datasources-configmap.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.grafana.enabled }}
+kind: ConfigMap
+metadata:
+  name: {{ template "lotus.fullname" . }}-grafana-datasources
+  labels:
+    lotus-grafana-datasource: "true"
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: thanos
+        type: prometheus
+        url: http://{{ template "lotus.fullname" . }}-thanos-query:9090
+        access: proxy
+        basicAuth: false
+        isDefault: true
+{{- end }}
diff --git a/install/helm/templates/prometheus-rbac.yaml b/install/helm/templates/prometheus-rbac.yaml
new file mode 100644
index 0000000..17c1347
--- /dev/null
+++ b/install/helm/templates/prometheus-rbac.yaml
@@ -0,0 +1,37 @@
+{{- if .Values.lotus.rbac.enabled }}
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-prometheus
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-prometheus
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "lotus.fullname" . }}-prometheus
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "lotus.fullname" . }}-prometheus
+subjects:
+- kind: ServiceAccount
+  name: {{ template "lotus.fullname" . }}-prometheus
+  namespace: {{ .Release.Namespace }}
+{{- end }}
\ No newline at end of file
diff --git a/install/helm/values.yaml b/install/helm/values.yaml
new file mode 100644
index 0000000..f1f68ca
--- /dev/null
+++ b/install/helm/values.yaml
@@ -0,0 +1,34 @@
+lotus:
+  image:
+    repository: nghialv2607/lotus
+    tag: v0.1.0
+  rbac:
+    enabled: true
+  configs:
+    checks:
+      - name: NoWorker
+        expr: absent(up)
+        for: 30s
+      - name: HasWorkerDown
+        expr: up == 0
+        for: 30s
+    receivers:
+      - name: logger
+        logger:
+    timeSeriesStorage:
+    grafanaBaseUrl: ""
+
+grafana:
+  enabled: true
+  replicas: 1
+  image:
+    repository: grafana/grafana
+    tag: 5.3.4
+  adminPassword: admin
+  sidecar:
+    dashboards:
+      enabled: true
+      label: lotus-grafana-dashboard
+    datasources:
+      enabled: true
+      label: lotus-grafana-datasource
diff --git a/install/manifest-generate-values-norbac.yaml b/install/manifest-generate-values-norbac.yaml
new file mode 100644
index 0000000..90c8caa
--- /dev/null
+++ b/install/manifest-generate-values-norbac.yaml
@@ -0,0 +1,18 @@
+lotus:
+  rbac:
+    enabled: false
+  # configs:
+  #   checks: FIX-ME---NOT-REQUIRED---SET-GLOBAL-CHECKS
+  #   receivers: FIX-ME---NOT-REQUIRED---SET-SUMMARY-RECEIVERS
+  #   timeSeriesStorage:
+  #     gcs:
+  #       bucket: FIX-ME---SET-BUCKET-TO-STORE-TIMESERIES
+  #       credentials:
+  #         secret: FIX-ME---SET-SECRET-NAME-THAT-CONTAINS-GCS-SERVICE-ACCOUNT
+  #         file: FIX-ME---SET-SERVICE-ACCOUNT-FILENAME
+
+grafana:
+  rbac:
+    create: false
+  serviceAccount:
+    create: false
diff --git a/install/manifest-generate-values.yaml b/install/manifest-generate-values.yaml
new file mode 100644
index 0000000..6ce80b8
--- /dev/null
+++ b/install/manifest-generate-values.yaml
@@ -0,0 +1,10 @@
+# lotus:
+#   configs:
+#     checks: FIX-ME---NOT-REQUIRED---SET-GLOBAL-CHECKS
+#     receivers: FIX-ME---NOT-REQUIRED---SET-SUMMARY-RECEIVERS
+#     timeSeriesStorage:
+#       gcs:
+#         bucket: FIX-ME---SET-BUCKET-TO-STORE-TIMESERIES
+#         credentials:
+#           secret: FIX-ME---SET-SECRET-NAME-THAT-CONTAINS-GCS-SERVICE-ACCOUNT
+#           file: FIX-ME---SET-SERVICE-ACCOUNT-FILENAME
\ No newline at end of file
diff --git a/install/manifests-norbac/controller-config-configmap.yaml b/install/manifests-norbac/controller-config-configmap.yaml
new file mode 100644
index 0000000..21cda9c
--- /dev/null
+++ b/install/manifests-norbac/controller-config-configmap.yaml
@@ -0,0 +1,21 @@
+---
+# Source: lotus/templates/controller-config-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lotus-controller-config
+data:
+  config.yaml: |
+    checks:
+    - expr: absent(up)
+      for: 30s
+      name: NoWorker
+    - expr: up == 0
+      for: 30s
+      name: HasWorkerDown
+    grafanaBaseUrl: ""
+    receivers:
+    - logger: null
+      name: logger
+    timeSeriesStorage: null
+    
diff --git a/install/manifests-norbac/controller-deployment.yaml b/install/manifests-norbac/controller-deployment.yaml
new file mode 100644
index 0000000..52512d6
--- /dev/null
+++ b/install/manifests-norbac/controller-deployment.yaml
@@ -0,0 +1,35 @@
+---
+# Source: lotus/templates/controller-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lotus-controller
+  labels:
+    app: lotus-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: lotus-controller
+  template:
+    metadata:
+      labels:
+        app: lotus-controller
+    spec:
+      containers:
+      - name: lotus-controller
+        image: nghialv2607/lotus:v0.1.0
+        args:
+        - controller
+        - --log-level=debug
+        - --config-file=/etc/lotus/config.yaml
+        - --namespace=default
+        - --release=lotus
+        volumeMounts:
+        - name: config
+          mountPath: /etc/lotus
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          name: lotus-controller-config
diff --git a/install/manifests-norbac/crd.yaml b/install/manifests-norbac/crd.yaml
new file mode 100644
index 0000000..8a1b497
--- /dev/null
+++ b/install/manifests-norbac/crd.yaml
@@ -0,0 +1,46 @@
+---
+# Source: lotus/templates/crd.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: lotuses.lotus.nghialv.com
+spec:
+  group: lotus.nghialv.com
+  version: v1beta1
+  scope: Namespaced
+  names:
+    kind: Lotus
+    plural: lotuses
+    singular: lotus
+    categories:
+      - all
+  additionalPrinterColumns:
+    - name: Phase
+      type: string
+      description: The current phase of Lotus
+      JSONPath: .status.phase
+    - name: WorkerReplicas
+      type: integer
+      description: The number of workers launched for this Lotus
+      JSONPath: .spec.worker.replicas
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          required:
+            - worker
+        status:
+          properties:
+            phase:
+              type: string
+              enum:
+              - "Pending"
+              - "Preparing"
+              - "Running"
+              - "Cleaning"
+              - "FailureCleaning"
+              - "Succeeded"
+              - "Failed"
diff --git a/install/manifests-norbac/grafana-configmap-dashboard-provider.yaml b/install/manifests-norbac/grafana-configmap-dashboard-provider.yaml
new file mode 100644
index 0000000..9831150
--- /dev/null
+++ b/install/manifests-norbac/grafana-configmap-dashboard-provider.yaml
@@ -0,0 +1,23 @@
+---
+# Source: lotus/charts/grafana/templates/configmap-dashboard-provider.yaml
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+  name: lotus-grafana-config-dashboards
+data:
+  provider.yaml: |-
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      options:
+        path: /tmp/dashboards
diff --git a/install/manifests-norbac/grafana-configmap.yaml b/install/manifests-norbac/grafana-configmap.yaml
new file mode 100644
index 0000000..e623c8e
--- /dev/null
+++ b/install/manifests-norbac/grafana-configmap.yaml
@@ -0,0 +1,24 @@
+---
+# Source: lotus/charts/grafana/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+data:
+  grafana.ini: |
+    [analytics]
+    check_for_updates = true
+    [grafana_net]
+    url = https://grafana.net
+    [log]
+    mode = console
+    [paths]
+    data = /var/lib/grafana/data
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
diff --git a/install/manifests-norbac/grafana-dashboards-configmap.yaml b/install/manifests-norbac/grafana-dashboards-configmap.yaml
new file mode 100644
index 0000000..e71ae0a
--- /dev/null
+++ b/install/manifests-norbac/grafana-dashboards-configmap.yaml
@@ -0,0 +1,1439 @@
+---
+# Source: lotus/templates/grafana-dashboards-configmap.yaml
+
+kind: ConfigMap
+metadata:
+  name: lotus-grafana-dashboards
+  labels:
+    lotus-grafana-dashboard: "true"
+data:
+  grpc-dashboard.json: |
+    {
+       "annotations": {
+          "list": [ ]
+       },
+       "editable": false,
+       "gnetId": null,
+       "graphTooltip": 0,
+       "hideControls": false,
+       "id": null,
+       "links": [ ],
+       "panels": [
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 0,
+                "y": 0
+             },
+             "id": 2,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum (up{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": " ",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of workers",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 12,
+                "y": 0
+             },
+             "id": 3,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "virtual_user_status",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of virtual users",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 6
+             },
+             "id": 4,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_per_second:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "RPCs / second",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 6
+             },
+             "id": 5,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_roundtrip_latency:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Latency",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 14
+             },
+             "id": 6,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_per_second:status{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_status }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "RPCs / seconds grouping by status",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 14
+             },
+             "id": 7,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_failure_percentage:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Percentage of failed RPCs",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 22
+             },
+             "id": 8,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_sent_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Sent Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 22
+             },
+             "id": 9,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_received_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Received Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          }
+       ],
+       "refresh": "",
+       "rows": [ ],
+       "schemaVersion": 16,
+       "style": "dark",
+       "tags": [
+          "grpc"
+       ],
+       "templating": {
+          "list": [
+             {
+                "allValue": null,
+                "current": { },
+                "datasource": "thanos",
+                "hide": 0,
+                "includeAll": false,
+                "label": "TestID",
+                "multi": false,
+                "name": "testId",
+                "options": [ ],
+                "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+                "refresh": 2,
+                "regex": "/\"(.*)-worker\"/",
+                "sort": 0,
+                "tagValuesQuery": "",
+                "tags": [ ],
+                "tagsQuery": "",
+                "type": "query",
+                "useTags": false
+             }
+          ]
+       },
+       "time": {
+          "from": "now-1h",
+          "to": "now"
+       },
+       "timepicker": {
+          "refresh_intervals": [
+             "5s",
+             "10s",
+             "30s",
+             "1m",
+             "5m",
+             "15m",
+             "30m",
+             "1h",
+             "2h",
+             "1d"
+          ],
+          "time_options": [
+             "5m",
+             "15m",
+             "1h",
+             "6h",
+             "12h",
+             "24h",
+             "2d",
+             "7d",
+             "30d"
+          ]
+       },
+       "timezone": "browser",
+       "title": "GRPC",
+       "version": 0
+    }
+  http-dashboard.json: |
+    {
+       "annotations": {
+          "list": [ ]
+       },
+       "editable": false,
+       "gnetId": null,
+       "graphTooltip": 0,
+       "hideControls": false,
+       "id": null,
+       "links": [ ],
+       "panels": [
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 0,
+                "y": 0
+             },
+             "id": 2,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum (up{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": " ",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of workers",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 12,
+                "y": 0
+             },
+             "id": 3,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "virtual_user_status",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of virtual users",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 6
+             },
+             "id": 4,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_completed_requests_per_second:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Requests / second",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 6
+             },
+             "id": 5,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_completed_requests_5xx_percentage:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Percentage of 5xx Requests",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 14
+             },
+             "id": 6,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_roundtrip_latency:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Latency",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 22
+             },
+             "id": 7,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_sent_bytes:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Sent Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 22
+             },
+             "id": 8,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_received_bytes:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Received Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          }
+       ],
+       "refresh": "",
+       "rows": [ ],
+       "schemaVersion": 16,
+       "style": "dark",
+       "tags": [
+          "http"
+       ],
+       "templating": {
+          "list": [
+             {
+                "allValue": null,
+                "current": { },
+                "datasource": "thanos",
+                "hide": 0,
+                "includeAll": false,
+                "label": "TestID",
+                "multi": false,
+                "name": "testId",
+                "options": [ ],
+                "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+                "refresh": 2,
+                "regex": "/\"(.*)-worker\"/",
+                "sort": 0,
+                "tagValuesQuery": "",
+                "tags": [ ],
+                "tagsQuery": "",
+                "type": "query",
+                "useTags": false
+             }
+          ]
+       },
+       "time": {
+          "from": "now-1h",
+          "to": "now"
+       },
+       "timepicker": {
+          "refresh_intervals": [
+             "5s",
+             "10s",
+             "30s",
+             "1m",
+             "5m",
+             "15m",
+             "30m",
+             "1h",
+             "2h",
+             "1d"
+          ],
+          "time_options": [
+             "5m",
+             "15m",
+             "1h",
+             "6h",
+             "12h",
+             "24h",
+             "2d",
+             "7d",
+             "30d"
+          ]
+       },
+       "timezone": "browser",
+       "title": "HTTP",
+       "version": 0
+    }
+  
\ No newline at end of file
diff --git a/install/manifests-norbac/grafana-datasources-configmap.yaml b/install/manifests-norbac/grafana-datasources-configmap.yaml
new file mode 100644
index 0000000..cda9bb1
--- /dev/null
+++ b/install/manifests-norbac/grafana-datasources-configmap.yaml
@@ -0,0 +1,18 @@
+---
+# Source: lotus/templates/grafana-datasources-configmap.yaml
+
+kind: ConfigMap
+metadata:
+  name: lotus-grafana-datasources
+  labels:
+    lotus-grafana-datasource: "true"
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: thanos
+        type: prometheus
+        url: http://lotus-thanos-query:9090
+        access: proxy
+        basicAuth: false
+        isDefault: true
diff --git a/install/manifests-norbac/grafana-deployment.yaml b/install/manifests-norbac/grafana-deployment.yaml
new file mode 100644
index 0000000..c645d51
--- /dev/null
+++ b/install/manifests-norbac/grafana-deployment.yaml
@@ -0,0 +1,132 @@
+---
+# Source: lotus/charts/grafana/templates/deployment.yaml
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+      release: lotus
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: grafana
+        release: lotus
+    spec:
+      serviceAccountName: default
+      securityContext:
+        fsGroup: 472
+        runAsUser: 472
+        
+      containers:
+        - name: grafana-sc-dashboard
+          image: "kiwigrid/k8s-sidecar:0.0.6"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LABEL
+              value: "lotus-grafana-dashboard"
+            - name: FOLDER
+              value: "/tmp/dashboards"
+          resources:
+            null
+            
+          volumeMounts:
+            - name: sc-dashboard-volume
+              mountPath: "/tmp/dashboards"
+        - name: grafana-sc-datasources
+          image: "kiwigrid/k8s-sidecar:0.0.6"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LABEL
+              value: "lotus-grafana-datasource"
+            - name: FOLDER
+              value: "/etc/grafana/provisioning/datasources"
+          resources:
+            null
+            
+          volumeMounts:
+            - name: sc-datasources-volume
+              mountPath: "/etc/grafana/provisioning/datasources"
+        - name: grafana
+          image: "grafana/grafana:5.3.4"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: config
+              mountPath: "/etc/grafana/grafana.ini"
+              subPath: grafana.ini
+            - name: ldap
+              mountPath: "/etc/grafana/ldap.toml"
+              subPath: ldap.toml
+            - name: storage
+              mountPath: "/var/lib/grafana"
+              subPath: 
+            - name: sc-dashboard-volume
+              mountPath: "/tmp/dashboards"
+            - name: sc-dashboard-provider
+              mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml"
+              subPath: provider.yaml
+            - name: sc-datasources-volume
+              mountPath: "/etc/grafana/provisioning/datasources"
+          ports:
+            - name: service
+              containerPort: 80
+              protocol: TCP
+            - name: grafana
+              containerPort: 3000
+              protocol: TCP
+          env:
+            - name: GF_SECURITY_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: lotus-grafana
+                  key: admin-user
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: lotus-grafana
+                  key: admin-password
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /api/health
+              port: 3000
+            initialDelaySeconds: 60
+            timeoutSeconds: 30
+            
+          readinessProbe:
+            httpGet:
+              path: /api/health
+              port: 3000
+            
+          resources:
+            {}
+            
+      volumes:
+        - name: config
+          configMap:
+            name: lotus-grafana
+        - name: ldap
+          secret:
+            secretName: lotus-grafana
+            items:
+              - key: ldap-toml
+                path: ldap.toml
+        - name: storage
+          emptyDir: {}
+        - name: sc-dashboard-volume
+          emptyDir: {}
+        - name: sc-dashboard-provider
+          configMap:
+            name: lotus-grafana-config-dashboards
+        - name: sc-datasources-volume
+          emptyDir: {}
diff --git a/install/manifests-norbac/grafana-podsecuritypolicy.yaml b/install/manifests-norbac/grafana-podsecuritypolicy.yaml
new file mode 100644
index 0000000..61a1a54
--- /dev/null
+++ b/install/manifests-norbac/grafana-podsecuritypolicy.yaml
@@ -0,0 +1,41 @@
+---
+# Source: lotus/charts/grafana/templates/podsecuritypolicy.yaml
+
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    heritage: Tiller
+    release: lotus
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false
diff --git a/install/manifests-norbac/grafana-secret.yaml b/install/manifests-norbac/grafana-secret.yaml
new file mode 100644
index 0000000..66da2bd
--- /dev/null
+++ b/install/manifests-norbac/grafana-secret.yaml
@@ -0,0 +1,16 @@
+---
+# Source: lotus/charts/grafana/templates/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+type: Opaque
+data:
+  admin-user: "YWRtaW4="
+  admin-password: "YWRtaW4="
+  ldap-toml: ""
diff --git a/install/manifests-norbac/grafana-service.yaml b/install/manifests-norbac/grafana-service.yaml
new file mode 100644
index 0000000..023964d
--- /dev/null
+++ b/install/manifests-norbac/grafana-service.yaml
@@ -0,0 +1,22 @@
+---
+# Source: lotus/charts/grafana/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+spec:
+  type: ClusterIP
+  ports:
+    - name: service
+      port: 80
+      protocol: TCP
+      targetPort: 3000
+
+  selector:
+    app: grafana
+    release: lotus
diff --git a/install/manifests/controller-config-configmap.yaml b/install/manifests/controller-config-configmap.yaml
new file mode 100644
index 0000000..21cda9c
--- /dev/null
+++ b/install/manifests/controller-config-configmap.yaml
@@ -0,0 +1,21 @@
+---
+# Source: lotus/templates/controller-config-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lotus-controller-config
+data:
+  config.yaml: |
+    checks:
+    - expr: absent(up)
+      for: 30s
+      name: NoWorker
+    - expr: up == 0
+      for: 30s
+      name: HasWorkerDown
+    grafanaBaseUrl: ""
+    receivers:
+    - logger: null
+      name: logger
+    timeSeriesStorage: null
+    
diff --git a/install/manifests/controller-deployment.yaml b/install/manifests/controller-deployment.yaml
new file mode 100644
index 0000000..adf58c1
--- /dev/null
+++ b/install/manifests/controller-deployment.yaml
@@ -0,0 +1,37 @@
+---
+# Source: lotus/templates/controller-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lotus-controller
+  labels:
+    app: lotus-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: lotus-controller
+  template:
+    metadata:
+      labels:
+        app: lotus-controller
+    spec:
+      serviceAccountName: lotus-controller
+      containers:
+      - name: lotus-controller
+        image: nghialv2607/lotus:v0.1.0
+        args:
+        - controller
+        - --log-level=debug
+        - --config-file=/etc/lotus/config.yaml
+        - --namespace=default
+        - --release=lotus
+        - --prometheus-service-account=lotus-prometheus
+        volumeMounts:
+        - name: config
+          mountPath: /etc/lotus
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          name: lotus-controller-config
diff --git a/install/manifests/controller-rbac.yaml b/install/manifests/controller-rbac.yaml
new file mode 100644
index 0000000..161e07d
--- /dev/null
+++ b/install/manifests/controller-rbac.yaml
@@ -0,0 +1,66 @@
+---
+# Source: lotus/templates/controller-rbac.yaml
+
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: lotus-controller
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lotus-controller
+  namespace: default
+rules:
+  - apiGroups:
+      - ""
+      - apps
+      - extensions
+    resources:
+      - pods
+      - deployments
+      - statefulsets
+      - services
+      - configmaps
+      - secrets
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - "lotus.nghialv.com"
+    resources:
+      - lotuses
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lotus-controller
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: lotus-controller
+subjects:
+- kind: ServiceAccount
+  name: lotus-controller
+  namespace: default
\ No newline at end of file
diff --git a/install/manifests/crd.yaml b/install/manifests/crd.yaml
new file mode 100644
index 0000000..8a1b497
--- /dev/null
+++ b/install/manifests/crd.yaml
@@ -0,0 +1,46 @@
+---
+# Source: lotus/templates/crd.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: lotuses.lotus.nghialv.com
+spec:
+  group: lotus.nghialv.com
+  version: v1beta1
+  scope: Namespaced
+  names:
+    kind: Lotus
+    plural: lotuses
+    singular: lotus
+    categories:
+      - all
+  additionalPrinterColumns:
+    - name: Phase
+      type: string
+      description: The current phase of Lotus
+      JSONPath: .status.phase
+    - name: WorkerReplicas
+      type: integer
+      description: The number of workers launched for this Lotus
+      JSONPath: .spec.worker.replicas
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          required:
+            - worker
+        status:
+          properties:
+            phase:
+              type: string
+              enum:
+              - "Pending"
+              - "Preparing"
+              - "Running"
+              - "Cleaning"
+              - "FailureCleaning"
+              - "Succeeded"
+              - "Failed"
diff --git a/install/manifests/grafana-clusterrole.yaml b/install/manifests/grafana-clusterrole.yaml
new file mode 100644
index 0000000..f87aefc
--- /dev/null
+++ b/install/manifests/grafana-clusterrole.yaml
@@ -0,0 +1,16 @@
+---
+# Source: lotus/charts/grafana/templates/clusterrole.yaml
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+  name: lotus-grafana-clusterrole
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["configmaps"]
+  verbs: ["get", "watch", "list"]
diff --git a/install/manifests/grafana-clusterrolebinding.yaml b/install/manifests/grafana-clusterrolebinding.yaml
new file mode 100644
index 0000000..2295c26
--- /dev/null
+++ b/install/manifests/grafana-clusterrolebinding.yaml
@@ -0,0 +1,20 @@
+---
+# Source: lotus/charts/grafana/templates/clusterrolebinding.yaml
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lotus-grafana-clusterrolebinding
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+subjects:
+  - kind: ServiceAccount
+    name: lotus-grafana
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: lotus-grafana-clusterrole
+  apiGroup: rbac.authorization.k8s.io
diff --git a/install/manifests/grafana-configmap-dashboard-provider.yaml b/install/manifests/grafana-configmap-dashboard-provider.yaml
new file mode 100644
index 0000000..9831150
--- /dev/null
+++ b/install/manifests/grafana-configmap-dashboard-provider.yaml
@@ -0,0 +1,23 @@
+---
+# Source: lotus/charts/grafana/templates/configmap-dashboard-provider.yaml
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+  name: lotus-grafana-config-dashboards
+data:
+  provider.yaml: |-
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      options:
+        path: /tmp/dashboards
diff --git a/install/manifests/grafana-configmap.yaml b/install/manifests/grafana-configmap.yaml
new file mode 100644
index 0000000..e623c8e
--- /dev/null
+++ b/install/manifests/grafana-configmap.yaml
@@ -0,0 +1,24 @@
+---
+# Source: lotus/charts/grafana/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+data:
+  grafana.ini: |
+    [analytics]
+    check_for_updates = true
+    [grafana_net]
+    url = https://grafana.net
+    [log]
+    mode = console
+    [paths]
+    data = /var/lib/grafana/data
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
diff --git a/install/manifests/grafana-dashboards-configmap.yaml b/install/manifests/grafana-dashboards-configmap.yaml
new file mode 100644
index 0000000..e71ae0a
--- /dev/null
+++ b/install/manifests/grafana-dashboards-configmap.yaml
@@ -0,0 +1,1439 @@
+---
+# Source: lotus/templates/grafana-dashboards-configmap.yaml
+
+kind: ConfigMap
+metadata:
+  name: lotus-grafana-dashboards
+  labels:
+    lotus-grafana-dashboard: "true"
+data:
+  grpc-dashboard.json: |
+    {
+       "annotations": {
+          "list": [ ]
+       },
+       "editable": false,
+       "gnetId": null,
+       "graphTooltip": 0,
+       "hideControls": false,
+       "id": null,
+       "links": [ ],
+       "panels": [
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 0,
+                "y": 0
+             },
+             "id": 2,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum (up{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": " ",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of workers",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 12,
+                "y": 0
+             },
+             "id": 3,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "virtual_user_status",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of virtual users",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 6
+             },
+             "id": 4,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_per_second:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "RPCs / second",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 6
+             },
+             "id": 5,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_roundtrip_latency:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Latency",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 14
+             },
+             "id": 6,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_per_second:status{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_status }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "RPCs / seconds grouping by status",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 14
+             },
+             "id": 7,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_completed_rpcs_failure_percentage:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Percentage of failed RPCs",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 22
+             },
+             "id": 8,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_sent_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Sent Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 22
+             },
+             "id": 9,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_grpc_client_received_bytes_per_rpc:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ grpc_client_method }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Received Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          }
+       ],
+       "refresh": "",
+       "rows": [ ],
+       "schemaVersion": 16,
+       "style": "dark",
+       "tags": [
+          "grpc"
+       ],
+       "templating": {
+          "list": [
+             {
+                "allValue": null,
+                "current": { },
+                "datasource": "thanos",
+                "hide": 0,
+                "includeAll": false,
+                "label": "TestID",
+                "multi": false,
+                "name": "testId",
+                "options": [ ],
+                "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+                "refresh": 2,
+                "regex": "/\"(.*)-worker\"/",
+                "sort": 0,
+                "tagValuesQuery": "",
+                "tags": [ ],
+                "tagsQuery": "",
+                "type": "query",
+                "useTags": false
+             }
+          ]
+       },
+       "time": {
+          "from": "now-1h",
+          "to": "now"
+       },
+       "timepicker": {
+          "refresh_intervals": [
+             "5s",
+             "10s",
+             "30s",
+             "1m",
+             "5m",
+             "15m",
+             "30m",
+             "1h",
+             "2h",
+             "1d"
+          ],
+          "time_options": [
+             "5m",
+             "15m",
+             "1h",
+             "6h",
+             "12h",
+             "24h",
+             "2d",
+             "7d",
+             "30d"
+          ]
+       },
+       "timezone": "browser",
+       "title": "GRPC",
+       "version": 0
+    }
+  http-dashboard.json: |
+    {
+       "annotations": {
+          "list": [ ]
+       },
+       "editable": false,
+       "gnetId": null,
+       "graphTooltip": 0,
+       "hideControls": false,
+       "id": null,
+       "links": [ ],
+       "panels": [
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 0,
+                "y": 0
+             },
+             "id": 2,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum (up{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": " ",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of workers",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 6,
+                "w": 12,
+                "x": 12,
+                "y": 0
+             },
+             "id": 3,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "sum by (virtual_user_status) (lotus_virtual_user_count{job=~\"$testId-worker\"})",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "virtual_user_status",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Number of virtual users",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 6
+             },
+             "id": 4,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_completed_requests_per_second:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Requests / second",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "short",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 6
+             },
+             "id": 5,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_completed_requests_5xx_percentage:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Percentage of 5xx Requests",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "percent",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 14
+             },
+             "id": 6,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_roundtrip_latency:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Latency",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "ms",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 22
+             },
+             "id": 7,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_sent_bytes:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Sent Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          },
+          {
+             "aliasColors": { },
+             "bars": false,
+             "dashLength": 10,
+             "dashes": false,
+             "datasource": "thanos",
+             "fill": 2,
+             "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 22
+             },
+             "id": 8,
+             "legend": {
+                "alignAsTable": true,
+                "avg": true,
+                "current": true,
+                "max": true,
+                "min": true,
+                "rightSide": false,
+                "show": true,
+                "sort": "current",
+                "sortDesc": true,
+                "total": false,
+                "values": true
+             },
+             "lines": true,
+             "linewidth": 2,
+             "links": [ ],
+             "nullPointMode": "null",
+             "percentage": false,
+             "pointradius": 5,
+             "points": false,
+             "renderer": "flot",
+             "repeat": null,
+             "seriesOverrides": [ ],
+             "spaceLength": 10,
+             "stack": false,
+             "steppedLine": false,
+             "targets": [
+                {
+                   "expr": "lotus_http_client_received_bytes:host:route:method{job=~\"$testId-worker\"}",
+                   "format": "time_series",
+                   "intervalFactor": 2,
+                   "legendFormat": "{{ http_client_method }}/{{ http_client_host }}{{ http_client_route }}",
+                   "refId": "A"
+                }
+             ],
+             "thresholds": [ ],
+             "timeFrom": null,
+             "timeShift": null,
+             "title": "Received Bytes",
+             "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+             },
+             "type": "graph",
+             "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": [ ]
+             },
+             "yaxes": [
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                },
+                {
+                   "format": "bytes",
+                   "label": null,
+                   "logBase": 1,
+                   "max": null,
+                   "min": null,
+                   "show": true
+                }
+             ]
+          }
+       ],
+       "refresh": "",
+       "rows": [ ],
+       "schemaVersion": 16,
+       "style": "dark",
+       "tags": [
+          "http"
+       ],
+       "templating": {
+          "list": [
+             {
+                "allValue": null,
+                "current": { },
+                "datasource": "thanos",
+                "hide": 0,
+                "includeAll": false,
+                "label": "TestID",
+                "multi": false,
+                "name": "testId",
+                "options": [ ],
+                "query": "query_result(count by(job) (count_over_time(up[$__range])))",
+                "refresh": 2,
+                "regex": "/\"(.*)-worker\"/",
+                "sort": 0,
+                "tagValuesQuery": "",
+                "tags": [ ],
+                "tagsQuery": "",
+                "type": "query",
+                "useTags": false
+             }
+          ]
+       },
+       "time": {
+          "from": "now-1h",
+          "to": "now"
+       },
+       "timepicker": {
+          "refresh_intervals": [
+             "5s",
+             "10s",
+             "30s",
+             "1m",
+             "5m",
+             "15m",
+             "30m",
+             "1h",
+             "2h",
+             "1d"
+          ],
+          "time_options": [
+             "5m",
+             "15m",
+             "1h",
+             "6h",
+             "12h",
+             "24h",
+             "2d",
+             "7d",
+             "30d"
+          ]
+       },
+       "timezone": "browser",
+       "title": "HTTP",
+       "version": 0
+    }
+  
\ No newline at end of file
diff --git a/install/manifests/grafana-datasources-configmap.yaml b/install/manifests/grafana-datasources-configmap.yaml
new file mode 100644
index 0000000..cda9bb1
--- /dev/null
+++ b/install/manifests/grafana-datasources-configmap.yaml
@@ -0,0 +1,18 @@
+---
+# Source: lotus/templates/grafana-datasources-configmap.yaml
+
+kind: ConfigMap
+metadata:
+  name: lotus-grafana-datasources
+  labels:
+    lotus-grafana-datasource: "true"
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: thanos
+        type: prometheus
+        url: http://lotus-thanos-query:9090
+        access: proxy
+        basicAuth: false
+        isDefault: true
diff --git a/install/manifests/grafana-deployment.yaml b/install/manifests/grafana-deployment.yaml
new file mode 100644
index 0000000..28d6c51
--- /dev/null
+++ b/install/manifests/grafana-deployment.yaml
@@ -0,0 +1,132 @@
+---
+# Source: lotus/charts/grafana/templates/deployment.yaml
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+      release: lotus
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: grafana
+        release: lotus
+    spec:
+      serviceAccountName: lotus-grafana
+      securityContext:
+        fsGroup: 472
+        runAsUser: 472
+        
+      containers:
+        - name: grafana-sc-dashboard
+          image: "kiwigrid/k8s-sidecar:0.0.6"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LABEL
+              value: "lotus-grafana-dashboard"
+            - name: FOLDER
+              value: "/tmp/dashboards"
+          resources:
+            null
+            
+          volumeMounts:
+            - name: sc-dashboard-volume
+              mountPath: "/tmp/dashboards"
+        - name: grafana-sc-datasources
+          image: "kiwigrid/k8s-sidecar:0.0.6"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: LABEL
+              value: "lotus-grafana-datasource"
+            - name: FOLDER
+              value: "/etc/grafana/provisioning/datasources"
+          resources:
+            null
+            
+          volumeMounts:
+            - name: sc-datasources-volume
+              mountPath: "/etc/grafana/provisioning/datasources"
+        - name: grafana
+          image: "grafana/grafana:5.3.4"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: config
+              mountPath: "/etc/grafana/grafana.ini"
+              subPath: grafana.ini
+            - name: ldap
+              mountPath: "/etc/grafana/ldap.toml"
+              subPath: ldap.toml
+            - name: storage
+              mountPath: "/var/lib/grafana"
+              subPath: 
+            - name: sc-dashboard-volume
+              mountPath: "/tmp/dashboards"
+            - name: sc-dashboard-provider
+              mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml"
+              subPath: provider.yaml
+            - name: sc-datasources-volume
+              mountPath: "/etc/grafana/provisioning/datasources"
+          ports:
+            - name: service
+              containerPort: 80
+              protocol: TCP
+            - name: grafana
+              containerPort: 3000
+              protocol: TCP
+          env:
+            - name: GF_SECURITY_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: lotus-grafana
+                  key: admin-user
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: lotus-grafana
+                  key: admin-password
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /api/health
+              port: 3000
+            initialDelaySeconds: 60
+            timeoutSeconds: 30
+            
+          readinessProbe:
+            httpGet:
+              path: /api/health
+              port: 3000
+            
+          resources:
+            {}
+            
+      volumes:
+        - name: config
+          configMap:
+            name: lotus-grafana
+        - name: ldap
+          secret:
+            secretName: lotus-grafana
+            items:
+              - key: ldap-toml
+                path: ldap.toml
+        - name: storage
+          emptyDir: {}
+        - name: sc-dashboard-volume
+          emptyDir: {}
+        - name: sc-dashboard-provider
+          configMap:
+            name: lotus-grafana-config-dashboards
+        - name: sc-datasources-volume
+          emptyDir: {}
diff --git a/install/manifests/grafana-podsecuritypolicy.yaml b/install/manifests/grafana-podsecuritypolicy.yaml
new file mode 100644
index 0000000..61a1a54
--- /dev/null
+++ b/install/manifests/grafana-podsecuritypolicy.yaml
@@ -0,0 +1,41 @@
+---
+# Source: lotus/charts/grafana/templates/podsecuritypolicy.yaml
+
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    heritage: Tiller
+    release: lotus
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false
diff --git a/install/manifests/grafana-role.yaml b/install/manifests/grafana-role.yaml
new file mode 100644
index 0000000..34f1668
--- /dev/null
+++ b/install/manifests/grafana-role.yaml
@@ -0,0 +1,17 @@
+---
+# Source: lotus/charts/grafana/templates/role.yaml
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    heritage: Tiller
+    release: lotus
+rules:
+- apiGroups:      ['extensions']
+  resources:      ['podsecuritypolicies']
+  verbs:          ['use']
+  resourceNames:  [lotus-grafana]
diff --git a/install/manifests/grafana-rolebinding.yaml b/install/manifests/grafana-rolebinding.yaml
new file mode 100644
index 0000000..2368a1a
--- /dev/null
+++ b/install/manifests/grafana-rolebinding.yaml
@@ -0,0 +1,18 @@
+---
+# Source: lotus/charts/grafana/templates/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    heritage: Tiller
+    release: lotus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: lotus-grafana
+subjects:
+- kind: ServiceAccount
+  name: lotus-grafana
\ No newline at end of file
diff --git a/install/manifests/grafana-secret.yaml b/install/manifests/grafana-secret.yaml
new file mode 100644
index 0000000..66da2bd
--- /dev/null
+++ b/install/manifests/grafana-secret.yaml
@@ -0,0 +1,16 @@
+---
+# Source: lotus/charts/grafana/templates/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+type: Opaque
+data:
+  admin-user: "YWRtaW4="
+  admin-password: "YWRtaW4="
+  ldap-toml: ""
diff --git a/install/manifests/grafana-service.yaml b/install/manifests/grafana-service.yaml
new file mode 100644
index 0000000..023964d
--- /dev/null
+++ b/install/manifests/grafana-service.yaml
@@ -0,0 +1,22 @@
+---
+# Source: lotus/charts/grafana/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: lotus-grafana
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    release: lotus
+    heritage: Tiller
+spec:
+  type: ClusterIP
+  ports:
+    - name: service
+      port: 80
+      protocol: TCP
+      targetPort: 3000
+
+  selector:
+    app: grafana
+    release: lotus
diff --git a/install/manifests/grafana-serviceaccount.yaml b/install/manifests/grafana-serviceaccount.yaml
new file mode 100644
index 0000000..914acc7
--- /dev/null
+++ b/install/manifests/grafana-serviceaccount.yaml
@@ -0,0 +1,12 @@
+---
+# Source: lotus/charts/grafana/templates/serviceaccount.yaml
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: grafana
+    chart: grafana-1.19.0
+    heritage: Tiller
+    release: lotus
+  name: lotus-grafana
diff --git a/install/manifests/prometheus-rbac.yaml b/install/manifests/prometheus-rbac.yaml
new file mode 100644
index 0000000..11a71a4
--- /dev/null
+++ b/install/manifests/prometheus-rbac.yaml
@@ -0,0 +1,38 @@
+---
+# Source: lotus/templates/prometheus-rbac.yaml
+
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: lotus-prometheus
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lotus-prometheus
+  namespace: default
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lotus-prometheus
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: lotus-prometheus
+subjects:
+- kind: ServiceAccount
+  name: lotus-prometheus
+  namespace: default
\ No newline at end of file
diff --git a/jsonnetfile.json b/jsonnetfile.json
new file mode 100644
index 0000000..86000e9
--- /dev/null
+++ b/jsonnetfile.json
@@ -0,0 +1,14 @@
+{
+    "dependencies": [
+        {
+            "name": "grafonnet",
+            "source": {
+                "git": {
+                    "remote": "https://github.com/grafana/grafonnet-lib",
+                    "subdir": "grafonnet"
+                }
+            },
+            "version": "master"
+        }
+    ]
+}
diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json
new file mode 100644
index 0000000..8a99886
--- /dev/null
+++ b/jsonnetfile.lock.json
@@ -0,0 +1,14 @@
+{
+    "dependencies": [
+        {
+            "name": "grafonnet",
+            "source": {
+                "git": {
+                    "remote": "https://github.com/grafana/grafonnet-lib",
+                    "subdir": "grafonnet"
+                }
+            },
+            "version": "eea8b5ba6b8883cf2df5a17c39a42c4b57c0d63e"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/libsonnet/grafonnet/alert_condition.libsonnet b/libsonnet/grafonnet/alert_condition.libsonnet
new file mode 100644
index 0000000..d70a1d3
--- /dev/null
+++ b/libsonnet/grafonnet/alert_condition.libsonnet
@@ -0,0 +1,44 @@
+{
+  /**
+   * Returns a new condition of alert of graph panel.
+   * Currently the only condition type that exists is a Query condition
+   * that allows to specify a query letter, time range and an aggregation function.
+   *
+   * @param evaluatorParams Value of threshold
+   * @param evaluatorType Type of threshold
+   * @param operatorType Operator between conditions
+   * @param queryRefId The letter defines what query to execute from the Metrics tab
+   * @param queryTimeStart Begging of time range
+   * @param queryTimeEnd End of time range
+   * @param reducerParams Params of an aggregation function
+   * @param reducerType Name of an aggregation function
+   * @return A json that represents a condition of alert
+   */
+  new(
+    evaluatorParams=[],
+    evaluatorType='gt',
+    operatorType='and',
+    queryRefId='A',
+    queryTimeEnd='now',
+    queryTimeStart='5m',
+    reducerParams=[],
+    reducerType='avg',
+  )::
+    {
+      evaluator: {
+        params: if std.type(evaluatorParams) == 'array' then evaluatorParams else [evaluatorParams],
+        type: evaluatorType,
+      },
+      operator: {
+        type: operatorType,
+      },
+      query: {
+        params: [queryRefId, queryTimeStart, queryTimeEnd],
+      },
+      reducer: {
+        params: if std.type(reducerParams) == 'array' then reducerParams else [reducerParams],
+        type: reducerType,
+      },
+      type: 'query',
+    },
+}
diff --git a/libsonnet/grafonnet/annotation.libsonnet b/libsonnet/grafonnet/annotation.libsonnet
new file mode 100644
index 0000000..653211e
--- /dev/null
+++ b/libsonnet/grafonnet/annotation.libsonnet
@@ -0,0 +1,35 @@
+{
+  default::
+    {
+      builtIn: 1,
+      datasource: '-- Grafana --',
+      enable: true,
+      hide: true,
+      iconColor: 'rgba(0, 211, 255, 1)',
+      name: 'Annotations & Alerts',
+      type: 'dashboard',
+    },
+  datasource(
+    name,
+    datasource,
+    expr=null,
+    enable=true,
+    hide=false,
+    iconColor='rgba(255, 96, 96, 1)',
+    tags=[],
+    type='tags',
+    builtIn=null,
+  )::
+    {
+      datasource: datasource,
+      enable: enable,
+      [if expr != null then 'expr']: expr,
+      hide: hide,
+      iconColor: iconColor,
+      name: name,
+      showIn: 0,
+      tags: tags,
+      type: type,
+      [if builtIn != null then 'builtIn']: builtIn,
+    },
+}
diff --git a/libsonnet/grafonnet/cloudwatch.libsonnet b/libsonnet/grafonnet/cloudwatch.libsonnet
new file mode 100644
index 0000000..6312eda
--- /dev/null
+++ b/libsonnet/grafonnet/cloudwatch.libsonnet
@@ -0,0 +1,39 @@
+{
+  /**
+   * Return a CloudWatch Target
+   *
+   * @param region
+   * @param namespace
+   * @param metric
+   * @param datasource
+   * @param statistic
+   * @param alias
+   * @param highResolution
+   * @param period
+   * @param dimensions
+
+   * @return Panel target
+   */
+
+  target(
+    region,
+    namespace,
+    metric,
+    datasource=null,
+    statistic='Average',
+    alias=null,
+    highResolution=false,
+    period='1m',
+    dimensions={}
+  ):: {
+    region: region,
+    namespace: namespace,
+    metricName: metric,
+    [if datasource != null then 'datasource']: datasource,
+    statistics: [statistic],
+    [if alias != null then 'alias']: alias,
+    highResolution: highResolution,
+    period: period,
+    dimensions: dimensions,
+  },
+}
diff --git a/libsonnet/grafonnet/dashboard.libsonnet b/libsonnet/grafonnet/dashboard.libsonnet
new file mode 100644
index 0000000..f07b548
--- /dev/null
+++ b/libsonnet/grafonnet/dashboard.libsonnet
@@ -0,0 +1,121 @@
+local timepickerlib = import 'timepicker.libsonnet';
+
+{
+  new(
+    title,
+    editable=false,
+    style='dark',
+    tags=[],
+    time_from='now-6h',
+    time_to='now',
+    timezone='browser',
+    refresh='',
+    timepicker=timepickerlib.new(),
+    graphTooltip='default',
+    hideControls=false,
+    schemaVersion=14,
+    uid='',
+    description=null,
+  ):: {
+    local it = self,
+    _annotations:: [],
+    [if uid != '' then 'uid']: uid,
+    editable: editable,
+    [if description != null then 'description']: description,
+    gnetId: null,
+    graphTooltip:
+      if graphTooltip == 'shared_tooltip' then 2
+      else if graphTooltip == 'shared_crosshair' then 1
+      else if graphTooltip == 'default' then 0
+      else graphTooltip,
+    hideControls: hideControls,
+    id: null,
+    links: [],
+    panels:: [],
+    refresh: refresh,
+    rows: [],
+    schemaVersion: schemaVersion,
+    style: style,
+    tags: tags,
+    time: {
+      from: time_from,
+      to: time_to,
+    },
+    timezone: timezone,
+    timepicker: timepicker,
+    title: title,
+    version: 0,
+    addAnnotation(annotation):: self {
+      _annotations+:: [annotation],
+    },
+    addTemplate(t):: self {
+      templates+: [t],
+    },
+    templates:: [],
+    annotations: { list: it._annotations },
+    templating: { list: it.templates },
+    _nextPanel:: 2,
+    addRow(row)::
+      self {
+        // automatically number panels in added rows.
+        // https://github.com/kausalco/public/blob/master/klumps/grafana.libsonnet
+        local n = std.length(row.panels),
+        local nextPanel = super._nextPanel,
+        local panels = std.makeArray(n, function(i)
+          row.panels[i] { id: nextPanel + i }),
+
+        _nextPanel: nextPanel + n,
+        rows+: [row { panels: panels }],
+      },
+    addPanels(newpanels)::
+      self {
+        // automatically number panels in added rows.
+        // https://github.com/kausalco/public/blob/master/klumps/grafana.libsonnet
+        local n = std.foldl(function(numOfPanels, p)
+          (if 'panels' in p then
+             numOfPanels + 1 + std.length(p.panels)
+           else
+             numOfPanels + 1), newpanels, 0),
+        local nextPanel = super._nextPanel,
+        local _panels = std.makeArray(
+          std.length(newpanels), function(i)
+            newpanels[i] {
+              id: nextPanel + (
+                if i == 0 then
+                  0
+                else
+                  if 'panels' in _panels[i - 1] then
+                    (_panels[i - 1].id - nextPanel) + 1 + std.length(_panels[i - 1].panels)
+                  else
+                    (_panels[i - 1].id - nextPanel) + 1
+
+              ),
+              [if 'panels' in newpanels[i] then 'panels']: std.makeArray(
+                std.length(newpanels[i].panels), function(j)
+                  newpanels[i].panels[j] {
+                    id: 1 + j +
+                        nextPanel + (
+                      if i == 0 then
+                        0
+                      else
+                        if 'panels' in _panels[i - 1] then
+                          (_panels[i - 1].id - nextPanel) + 1 + std.length(_panels[i - 1].panels)
+                        else
+                          (_panels[i - 1].id - nextPanel) + 1
+
+                    ),
+                  }
+              ),
+            }
+        ),
+
+        _nextPanel: nextPanel + n,
+        panels+::: _panels,
+      },
+    addPanel(panel, gridPos):: self + self.addPanels([panel { gridPos: gridPos }]),
+    addRows(rows):: std.foldl(function(d, row) d.addRow(row), rows, self),
+    addLink(link):: self {
+      links+: [link],
+    },
+  },
+}
diff --git a/libsonnet/grafonnet/elasticsearch.libsonnet b/libsonnet/grafonnet/elasticsearch.libsonnet
new file mode 100644
index 0000000..1f87762
--- /dev/null
+++ b/libsonnet/grafonnet/elasticsearch.libsonnet
@@ -0,0 +1,38 @@
+{
+  target(
+    query,
+    id=null,
+    datasource=null,
+    metrics=[{
+      field: "value",
+      id: null,
+      type: "percentiles",
+      settings: {
+        percents: [
+          "90",
+        ],
+      },
+    }],
+    bucketAggs=[{
+      field: "timestamp",
+      id: null,
+      type: "date_histogram",
+      settings: {
+          interval: "1s",
+          min_doc_count: 0,
+          trimEdges: 0
+      },
+    }],
+    timeField,
+    alias=null,
+  ):: {
+    [if datasource != null then 'datasource']: datasource,
+    query: query,
+    id: id,
+    timeField: timeField,
+    bucketAggs: bucketAggs,
+    metrics: metrics,
+    alias: alias
+    // TODO: generate bucket ids
+  }
+}
diff --git a/libsonnet/grafonnet/grafana.libsonnet b/libsonnet/grafonnet/grafana.libsonnet
new file mode 100644
index 0000000..5b7e7d3
--- /dev/null
+++ b/libsonnet/grafonnet/grafana.libsonnet
@@ -0,0 +1,19 @@
+{
+  dashboard:: import 'dashboard.libsonnet',
+  template:: import 'template.libsonnet',
+  text:: import 'text.libsonnet',
+  timepicker:: import 'timepicker.libsonnet',
+  row:: import 'row.libsonnet',
+  link:: import 'link.libsonnet',
+  annotation:: import 'annotation.libsonnet',
+  graphPanel:: import 'graph_panel.libsonnet',
+  tablePanel:: import 'table_panel.libsonnet',
+  singlestat:: import 'singlestat.libsonnet',
+  influxdb:: import 'influxdb.libsonnet',
+  prometheus:: import 'prometheus.libsonnet',
+  sql:: import 'sql.libsonnet',
+  graphite:: import 'graphite.libsonnet',
+  alertCondition:: import 'alert_condition.libsonnet',
+  cloudwatch:: import 'cloudwatch.libsonnet',
+  elasticsearch:: import 'elasticsearch.libsonnet',
+}
diff --git a/libsonnet/grafonnet/graph_panel.libsonnet b/libsonnet/grafonnet/graph_panel.libsonnet
new file mode 100644
index 0000000..6df2ddf
--- /dev/null
+++ b/libsonnet/grafonnet/graph_panel.libsonnet
@@ -0,0 +1,232 @@
+{
+  /**
+   * Returns a new graph panel that can be added in a row.
+   * It requires the graph panel plugin in grafana, which is built-in.
+   *
+   * @param title The title of the graph panel.
+   * @param span Width of the panel
+   * @param datasource Datasource
+   * @param fill Fill, integer from 0 to 10
+   * @param linewidth Line Width, integer from 0 to 10
+   * @param decimals Override automatic decimal precision for legend and tooltip. If null, not added to the json output.
+   * @param min_span Min span
+   * @param format Unit of the Y axes
+   * @param formatY1 Unit of the first Y axe
+   * @param formatY2 Unit of the second Y axe
+   * @param min Min of the Y axes
+   * @param max Max of the Y axes
+   * @param x_axis_mode X axis mode, one of [time, series, histogram]
+   * @param x_axis_values Chosen value of series, one of [avg, min, max, total, count]
+   * @param lines Display lines, boolean
+   * @param points Display points, boolean
+   * @param pointradius Radius of the points, allowed values are 0.5 or [1 ... 10] with step 1
+   * @param bars Display bars, boolean
+   * @param dashes Display line as dashes
+   * @param stack Stack values
+   * @param repeat Variable used to repeat the graph panel
+   * @param legend_show Show legend
+   * @param legend_values Show values in legend
+   * @param legend_min Show min in legend
+   * @param legend_max Show max in legend
+   * @param legend_current Show current in legend
+   * @param legend_total Show total in legend
+   * @param legend_avg Show average in legend
+   * @param legend_alignAsTable Show legend as table
+   * @param legend_rightSide Show legend to the right
+   * @param legend_sort Sort order of legend
+   * @param legend_sortDesc Sort legend descending
+   * @param aliasColors Define color mappings for graphs
+   * @param valueType Type of tooltip value
+   * @param thresholds Configuration of graph thresholds
+   * @param logBase1Y Value of logarithm base of the first Y axe
+   * @param logBase2Y Value of logarithm base of the second Y axe
+   * @param transparent Boolean (default: false) If set to true the panel will be transparent
+   * @return A json that represents a graph panel
+   */
+  new(
+    title,
+    span=null,
+    fill=1,
+    linewidth=1,
+    decimals=null,
+    description=null,
+    min_span=null,
+    format='short',
+    formatY1=null,
+    formatY2=null,
+    min=null,
+    max=null,
+    x_axis_mode='time',
+    x_axis_values='total',
+    lines=true,
+    datasource=null,
+    points=false,
+    pointradius=5,
+    bars=false,
+    height=null,
+    nullPointMode='null',
+    dashes=false,
+    stack=false,
+    repeat=null,
+    repeatDirection=null,
+    sort=0,
+    show_xaxis=true,
+    legend_show=true,
+    legend_values=false,
+    legend_min=false,
+    legend_max=false,
+    legend_current=false,
+    legend_total=false,
+    legend_avg=false,
+    legend_alignAsTable=false,
+    legend_rightSide=false,
+    legend_hideEmpty=null,
+    legend_hideZero=null,
+    legend_sort=null,
+    legend_sortDesc=null,
+    aliasColors={},
+    thresholds=[],
+    logBase1Y=1,
+    logBase2Y=1,
+    transparent=false,
+    value_type='individual'
+  ):: {
+    title: title,
+    [if span != null then 'span']: span,
+    [if min_span != null then 'minSpan']: min_span,
+    type: 'graph',
+    datasource: datasource,
+    targets: [
+    ],
+    [if description != null then 'description']: description,
+    [if height != null then 'height']: height,
+    renderer: 'flot',
+    yaxes: [
+      self.yaxe(if formatY1 != null then formatY1 else format, min, max, decimals=decimals, logBase=logBase1Y),
+      self.yaxe(if formatY2 != null then formatY2 else format, min, max, decimals=decimals, logBase=logBase2Y),
+    ],
+    xaxis: {
+      show: show_xaxis,
+      mode: x_axis_mode,
+      name: null,
+      values: if x_axis_mode == 'series' then [x_axis_values] else [],
+      buckets: null,
+    },
+    lines: lines,
+    fill: fill,
+    linewidth: linewidth,
+    dashes: dashes,
+    dashLength: 10,
+    spaceLength: 10,
+    points: points,
+    pointradius: pointradius,
+    bars: bars,
+    stack: stack,
+    percentage: false,
+    legend: {
+      show: legend_show,
+      values: legend_values,
+      min: legend_min,
+      max: legend_max,
+      current: legend_current,
+      total: legend_total,
+      alignAsTable: legend_alignAsTable,
+      rightSide: legend_rightSide,
+      avg: legend_avg,
+      [if legend_hideEmpty != null then 'hideEmpty']: legend_hideEmpty,
+      [if legend_hideZero != null then 'hideZero']: legend_hideZero,
+      [if legend_sort != null then 'sort']: legend_sort,
+      [if legend_sortDesc != null then 'sortDesc']: legend_sortDesc,
+    },
+    nullPointMode: nullPointMode,
+    steppedLine: false,
+    tooltip: {
+      value_type: value_type,
+      shared: true,
+      sort: if sort == 'decreasing' then 2 else if sort == 'increasing' then 1 else sort,
+    },
+    timeFrom: null,
+    timeShift: null,
+    [if transparent == true then 'transparent']: transparent,
+    aliasColors: aliasColors,
+    repeat: repeat,
+    [if repeatDirection != null then 'repeatDirection']: repeatDirection,
+    seriesOverrides: [],
+    thresholds: thresholds,
+    links: [],
+    yaxe(
+      format='short',
+      min=null,
+      max=null,
+      label=null,
+      show=true,
+      logBase=1,
+      decimals=null,
+    ):: {
+      label: label,
+      show: show,
+      logBase: logBase,
+      min: min,
+      max: max,
+      format: format,
+      [if decimals != null then 'decimals']: decimals,
+    },
+    _nextTarget:: 0,
+    addTarget(target):: self {
+      // automatically ref id in added targets.
+      // https://github.com/kausalco/public/blob/master/klumps/grafana.libsonnet
+      local nextTarget = super._nextTarget,
+      _nextTarget: nextTarget + 1,
+      targets+: [target { refId: std.char(std.codepoint('A') + nextTarget) }],
+    },
+    addTargets(targets):: std.foldl(function(p, t) p.addTarget(t), targets, self),
+    _nextSeriesOverride:: 0,
+    addSeriesOverride(override):: self {
+      local nextOverride = super._nextSerieOverride,
+      _nextSeriesOverride: nextOverride + 1,
+      seriesOverrides+: [override],
+    },
+    resetYaxes():: self {
+      yaxes: [],
+      _nextYaxis:: 0,
+    },
+    _nextYaxis:: 0,
+    addYaxis(
+      format='short',
+      min=null,
+      max=null,
+      label=null,
+      show=true,
+      logBase=1,
+      decimals=null,
+    ):: self {
+      local nextYaxis = super._nextYaxis,
+      _nextYaxis: nextYaxis + 1,
+      yaxes+: [self.yaxe(format, min, max, label, show, logBase, decimals)],
+    },
+    addAlert(
+      name,
+      executionErrorState='alerting',
+      frequency='60s',
+      handler=1,
+      noDataState='no_data',
+      notifications=[],
+    ):: self {
+      local it = self,
+      _conditions:: [],
+      alert: {
+        name: name,
+        conditions: it._conditions,
+        executionErrorState: executionErrorState,
+        frequency: frequency,
+        handler: handler,
+        noDataState: noDataState,
+        notifications: notifications,
+      },
+      addCondition(condition):: self {
+        _conditions+: [condition],
+      },
+      addConditions(conditions):: std.foldl(function(p, c) p.addCondition(c), conditions, it),
+    },
+  },
+}
diff --git a/libsonnet/grafonnet/graphite.libsonnet b/libsonnet/grafonnet/graphite.libsonnet
new file mode 100644
index 0000000..15e20eb
--- /dev/null
+++ b/libsonnet/grafonnet/graphite.libsonnet
@@ -0,0 +1,27 @@
+{
+  /**
+   * Return an Graphite Target
+   *
+   * @param target Graphite Query. Nested queries are possible by adding the query reference (refId).
+   * @param targetFull Expanding the @target. Used in nested queries.
+   * @param hide Disable query on graph.
+   * @param textEditor Enable raw query mode.
+   * @param datasource Datasource.
+
+   * @return Panel target
+   */
+  target(
+    target,
+    targetFull=null,
+    hide=false,
+    textEditor=false,
+    datasource=null,
+  ):: {
+    target: target,
+    hide: hide,
+    textEditor: textEditor,
+
+    [if targetFull != null then 'targetFull']: targetFull,
+    [if datasource != null then 'datasource']: datasource,
+  },
+}
diff --git a/libsonnet/grafonnet/influxdb.libsonnet b/libsonnet/grafonnet/influxdb.libsonnet
new file mode 100644
index 0000000..8e0451f
--- /dev/null
+++ b/libsonnet/grafonnet/influxdb.libsonnet
@@ -0,0 +1,27 @@
+{
+  /**
+   * Return an InfluxDB Target
+   *
+   * @param query Raw InfluxQL statement
+   * @param alias Alias By pattern
+   * @param datasource Datasource
+   * @param rawQuery En/Disable raw query mode
+   * @param resultFormat Format results as 'Time series' or 'Table'
+
+   * @return Panel target
+   */
+  target(
+    query,
+    alias=null,
+    datasource=null,
+    rawQuery=true,
+    resultFormat='time_series',
+  ):: {
+    query: query,
+    rawQuery: rawQuery,
+    resultFormat: resultFormat,
+
+    [if alias != null then 'alias']: alias,
+    [if datasource != null then 'datasource']: datasource,
+  },
+}
diff --git a/libsonnet/grafonnet/link.libsonnet b/libsonnet/grafonnet/link.libsonnet
new file mode 100644
index 0000000..97db18a
--- /dev/null
+++ b/libsonnet/grafonnet/link.libsonnet
@@ -0,0 +1,24 @@
+{
+  dashboards(
+    title,
+    tags,
+    asDropdown=true,
+    includeVars=false,
+    keepTime=false,
+    icon='external link',
+    url='',
+    targetBlank=false,
+    type='dashboards',
+  )::
+    {
+      asDropdown: asDropdown,
+      icon: icon,
+      includeVars: includeVars,
+      keepTime: keepTime,
+      tags: tags,
+      title: title,
+      type: type,
+      url: url,
+      targetBlank: targetBlank,
+    },
+}
diff --git a/libsonnet/grafonnet/prometheus.libsonnet b/libsonnet/grafonnet/prometheus.libsonnet
new file mode 100644
index 0000000..a15645d
--- /dev/null
+++ b/libsonnet/grafonnet/prometheus.libsonnet
@@ -0,0 +1,19 @@
+{
+  target(
+    expr,
+    format='time_series',
+    intervalFactor=2,
+    legendFormat='',
+    datasource=null,
+    interval=null,
+    instant=null,
+  ):: {
+    [if datasource != null then 'datasource']: datasource,
+    expr: expr,
+    format: format,
+    intervalFactor: intervalFactor,
+    legendFormat: legendFormat,
+    [if interval != null then 'interval']: interval,
+    [if instant != null then 'instant']: instant,
+  },
+}
diff --git a/libsonnet/grafonnet/row.libsonnet b/libsonnet/grafonnet/row.libsonnet
new file mode 100644
index 0000000..f5eb6c7
--- /dev/null
+++ b/libsonnet/grafonnet/row.libsonnet
@@ -0,0 +1,32 @@
+{
+  new(
+    title='Dashboard Row',
+    height=null,
+    collapse=false,
+    repeat=null,
+    showTitle=null,
+    titleSize='h6'
+  ):: {
+    collapse: collapse,
+    collapsed: collapse,
+    [if height != null then 'height']: height,
+    panels: [],
+    repeat: repeat,
+    repeatIteration: null,
+    repeatRowId: null,
+    showTitle:
+      if showTitle != null then
+        showTitle
+      else
+        title != 'Dashboard Row',
+    title: title,
+    type: 'row',
+    titleSize: titleSize,
+    addPanels(panels):: self {
+      panels+: panels,
+    },
+    addPanel(panel, gridPos={}):: self {
+      panels+: [panel { gridPos: gridPos }],
+    },
+  },
+}
diff --git a/libsonnet/grafonnet/singlestat.libsonnet b/libsonnet/grafonnet/singlestat.libsonnet
new file mode 100644
index 0000000..23d838f
--- /dev/null
+++ b/libsonnet/grafonnet/singlestat.libsonnet
@@ -0,0 +1,127 @@
+{
+  new(
+    title,
+    format='none',
+    description='',
+    interval=null,
+    height=null,
+    datasource=null,
+    span=null,
+    min_span=null,
+    decimals=null,
+    valueName='avg',
+    valueFontSize='80%',
+    prefixFontSize='50%',
+    postfixFontSize='50%',
+    mappingType=1,
+    repeat=null,
+    repeatDirection=null,
+    prefix='',
+    postfix='',
+    colors=[
+      '#299c46',
+      'rgba(237, 129, 40, 0.89)',
+      '#d44a3a',
+    ],
+    colorBackground=false,
+    colorValue=false,
+    thresholds='',
+    valueMaps=[
+      {
+        value: 'null',
+        op: '=',
+        text: 'N/A',
+      },
+    ],
+    rangeMaps=[
+      {
+        from: 'null',
+        to: 'null',
+        text: 'N/A',
+      },
+    ],
+    transparent=null,
+    sparklineFillColor='rgba(31, 118, 189, 0.18)',
+    sparklineFull=false,
+    sparklineLineColor='rgb(31, 120, 193)',
+    sparklineShow=false,
+    gaugeShow=false,
+    gaugeMinValue=0,
+    gaugeMaxValue=100,
+    gaugeThresholdMarkers=true,
+    gaugeThresholdLabels=false,
+  )::
+    {
+      [if height != null then 'height']: height,
+      [if description != '' then 'description']: description,
+      [if repeat != null then 'repeat']: repeat,
+      [if repeatDirection != null then 'repeatDirection']: repeatDirection,
+      [if transparent != null then 'transparent']: transparent,
+      [if min_span != null then 'minSpan']: min_span,
+      title: title,
+      [if span != null then 'span']: span,
+      type: 'singlestat',
+      datasource: datasource,
+      targets: [
+      ],
+      links: [],
+      [if decimals != null then 'decimals']: decimals,
+      maxDataPoints: 100,
+      interval: interval,
+      cacheTimeout: null,
+      format: format,
+      prefix: prefix,
+      postfix: postfix,
+      nullText: null,
+      valueMaps: valueMaps,
+      mappingTypes: [
+        {
+          name: 'value to text',
+          value: 1,
+        },
+        {
+          name: 'range to text',
+          value: 2,
+        },
+      ],
+      rangeMaps: rangeMaps,
+      mappingType:
+        if mappingType == 'value'
+        then
+          1
+        else if mappingType == 'range'
+        then
+          2
+        else
+          mappingType,
+      nullPointMode: 'connected',
+      valueName: valueName,
+      prefixFontSize: prefixFontSize,
+      valueFontSize: valueFontSize,
+      postfixFontSize: postfixFontSize,
+      thresholds: thresholds,
+      colorBackground: colorBackground,
+      colorValue: colorValue,
+      colors: colors,
+      gauge: {
+        show: gaugeShow,
+        minValue: gaugeMinValue,
+        maxValue: gaugeMaxValue,
+        thresholdMarkers: gaugeThresholdMarkers,
+        thresholdLabels: gaugeThresholdLabels,
+      },
+      sparkline: {
+        fillColor: sparklineFillColor,
+        full: sparklineFull,
+        lineColor: sparklineLineColor,
+        show: sparklineShow,
+      },
+      tableColumn: '',
+      _nextTarget:: 0,
+      addTarget(target):: self {
+        local nextTarget = super._nextTarget,
+        _nextTarget: nextTarget + 1,
+        targets+: [target { refId: std.char(std.codepoint('A') + nextTarget) }],
+      },
+    },
+}
diff --git a/libsonnet/grafonnet/sql.libsonnet b/libsonnet/grafonnet/sql.libsonnet
new file mode 100644
index 0000000..22229f6
--- /dev/null
+++ b/libsonnet/grafonnet/sql.libsonnet
@@ -0,0 +1,11 @@
+{
+  target(
+    rawSql,
+    datasource=null,
+    format='time_series',
+  ):: {
+    [if datasource != null then 'datasource']: datasource,
+    format: format,
+    rawSql: rawSql,
+  },
+}
diff --git a/libsonnet/grafonnet/table_panel.libsonnet b/libsonnet/grafonnet/table_panel.libsonnet
new file mode 100644
index 0000000..be32211
--- /dev/null
+++ b/libsonnet/grafonnet/table_panel.libsonnet
@@ -0,0 +1,41 @@
+{
+  /**
+   * Returns a new table panel that can be added in a row.
+   * It requires the table panel plugin in grafana, which is built-in.
+   *
+   * @param title The title of the graph panel.
+   * @param span Width of the panel
+   * @param description Description of the panel
+   * @param datasource Datasource
+   * @param min_span Min span
+   * @param styles Styles for the panel
+   * @return A json that represents a table panel
+   */
+  new(
+    title,
+    description=null,
+    span=null,
+    min_span=null,
+    datasource=null,
+    styles=[],
+  ):: {
+    type: 'table',
+    title: title,
+    [if span != null then 'span']: span,
+    [if min_span != null then 'minSpan']: min_span,
+    datasource: datasource,
+    targets: [
+    ],
+    styles: styles,
+    [if description != null then 'description']: description,
+    transform: 'table',
+    _nextTarget:: 0,
+    addTarget(target):: self {
+      // automatically ref id in added targets.
+      // https://github.com/kausalco/public/blob/master/klumps/grafana.libsonnet
+      local nextTarget = super._nextTarget,
+      _nextTarget: nextTarget + 1,
+      targets+: [target { refId: std.char(std.codepoint('A') + nextTarget) }],
+    },
+  },
+}
diff --git a/libsonnet/grafonnet/template.libsonnet b/libsonnet/grafonnet/template.libsonnet
new file mode 100644
index 0000000..afc22cd
--- /dev/null
+++ b/libsonnet/grafonnet/template.libsonnet
@@ -0,0 +1,131 @@
+{
+  new(
+    name,
+    datasource,
+    query,
+    label=null,
+    allValues=null,
+    tagValuesQuery='',
+    current=null,
+    hide='',
+    regex='',
+    refresh='never',
+    includeAll=false,
+    multi=false,
+    sort=0,
+  )::
+    {
+      allValue: allValues,
+      current: $.current(current),
+      datasource: datasource,
+      includeAll: includeAll,
+      hide: $.hide(hide),
+      label: label,
+      multi: multi,
+      name: name,
+      options: [],
+      query: query,
+      refresh: $.refresh(refresh),
+      regex: regex,
+      sort: sort,
+      tagValuesQuery: tagValuesQuery,
+      tags: [],
+      tagsQuery: '',
+      type: 'query',
+      useTags: false,
+    },
+  interval(
+    name,
+    query,
+    current,
+    hide='',
+    label=null,
+    auto_count=300,
+    auto_min='10s',
+  )::
+    {
+      current: $.current(current),
+      hide: if hide == '' then 0 else if hide == 'label' then 1 else 2,
+      label: label,
+      name: name,
+      query: std.join(',', std.filter($.filterAuto, std.split(query, ','))),
+      refresh: 2,
+      type: 'interval',
+      auto: std.count(std.split(query, ','), 'auto') > 0,
+      auto_count: auto_count,
+      auto_min: auto_min,
+    },
+  hide(hide)::
+    if hide == '' then 0 else if hide == 'label' then 1 else 2,
+  current(current):: {
+    [if current != null then 'text']: current,
+    [if current != null then 'value']: if current == 'auto' then
+      '$__auto_interval'
+    else if current == 'all' then
+      '$__all'
+    else
+      current,
+  },
+  datasource(
+    name,
+    query,
+    current,
+    hide='',
+    label=null,
+    regex='',
+    refresh='load',
+  ):: {
+    current: $.current(current),
+    hide: $.hide(hide),
+    label: label,
+    name: name,
+    options: [],
+    query: query,
+    refresh: $.refresh(refresh),
+    regex: regex,
+    type: 'datasource',
+  },
+  refresh(refresh):: if refresh == 'never'
+  then
+    0
+  else if refresh == 'load'
+  then
+    1
+  else if refresh == 'time'
+  then
+    2
+  else
+    refresh,
+  filterAuto(str):: str != 'auto',
+  custom(
+    name,
+    query,
+    current,
+    refresh='never',
+    label='',
+    valuelabels={},
+    hide='',
+  )::
+    {
+      allValue: null,
+      current: {
+        value: current,
+        text: if current in valuelabels then valuelabels[current] else current,
+      },
+      options: std.map(
+        function(i)
+          {
+            text: if i in valuelabels then valuelabels[i] else i,
+            value: i,
+          }, std.split(query, ',')
+      ),
+      hide: $.hide(hide),
+      includeAll: false,
+      label: label,
+      refresh: $.refresh(refresh),
+      multi: false,
+      name: name,
+      query: query,
+      type: 'custom',
+    },
+}
diff --git a/libsonnet/grafonnet/text.libsonnet b/libsonnet/grafonnet/text.libsonnet
new file mode 100644
index 0000000..757d693
--- /dev/null
+++ b/libsonnet/grafonnet/text.libsonnet
@@ -0,0 +1,17 @@
+{
+  new(
+    title='',
+    span=null,
+    mode='markdown',
+    content='',
+    transparent=null,
+  )::
+    {
+      [if transparent != null then 'transparent']: transparent,
+      title: title,
+      [if span != null then 'span']: span,
+      type: 'text',
+      mode: mode,
+      content: content,
+    },
+}
diff --git a/libsonnet/grafonnet/timepicker.libsonnet b/libsonnet/grafonnet/timepicker.libsonnet
new file mode 100644
index 0000000..0d51d2e
--- /dev/null
+++ b/libsonnet/grafonnet/timepicker.libsonnet
@@ -0,0 +1,30 @@
+{
+  new(
+    refresh_intervals=[
+      '5s',
+      '10s',
+      '30s',
+      '1m',
+      '5m',
+      '15m',
+      '30m',
+      '1h',
+      '2h',
+      '1d',
+    ],
+    time_options=[
+      '5m',
+      '15m',
+      '1h',
+      '6h',
+      '12h',
+      '24h',
+      '2d',
+      '7d',
+      '30d',
+    ],
+  ):: {
+    refresh_intervals: refresh_intervals,
+    time_options: time_options,
+  },
+}
diff --git a/pgv_proto_library.bzl b/pgv_proto_library.bzl
new file mode 100644
index 0000000..8cecedb
--- /dev/null
+++ b/pgv_proto_library.bzl
@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+def pgv_go_proto_library(name, proto = None, deps = [], **kwargs):
+    go_proto_compiler(
+        name = "pgv_plugin_go",
+        suffix = ".pb.validate.go",
+        valid_archive = False,
+        plugin = "@com_lyft_protoc_gen_validate//:protoc-gen-validate",
+        options = ["lang=go"],
+    )
+
+    go_proto_library(
+        name = name,
+        proto = proto,
+        deps = ["@com_lyft_protoc_gen_validate//validate:go_default_library"] + deps,
+        compilers = [
+            "@io_bazel_rules_go//proto:go_proto",
+            "pgv_plugin_go",
+        ],
+        visibility = ["//visibility:public"],
+        **kwargs
+    )
+
diff --git a/pkg/app/example/cmd/helloworld/BUILD.bazel b/pkg/app/example/cmd/helloworld/BUILD.bazel
new file mode 100644
index 0000000..3e07de3
--- /dev/null
+++ b/pkg/app/example/cmd/helloworld/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["helloworld.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/cmd/helloworld",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/example/helloworld:go_default_library",
+        "//pkg/cli:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+        "@org_golang_google_grpc//status:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/example/cmd/helloworld/helloworld.go b/pkg/app/example/cmd/helloworld/helloworld.go
new file mode 100644
index 0000000..855224b
--- /dev/null
+++ b/pkg/app/example/cmd/helloworld/helloworld.go
@@ -0,0 +1,122 @@
+package helloworld
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	helloworldproto "github.com/nghialv/lotus/pkg/app/example/helloworld"
+	"github.com/nghialv/lotus/pkg/cli"
+)
+
+type server struct {
+	grpcPort int
+	httpPort int
+}
+
+func NewCommand() *cobra.Command {
+	s := &server{
+		grpcPort: 8080,
+		httpPort: 9090,
+	}
+	cmd := &cobra.Command{
+		Use:   "helloworld",
+		Short: "Start running helloworld server",
+		RunE:  cli.WithContext(s.run),
+	}
+	cmd.Flags().IntVar(&s.grpcPort, "grpc-port", s.grpcPort, "Port number used to expose grpc server")
+	cmd.Flags().IntVar(&s.httpPort, "http-port", s.httpPort, "Port number used to expose http server")
+	return cmd
+}
+
+func (s *server) run(ctx context.Context, logger *zap.Logger) error {
+	// Start a grpc server to handle grpc calls defined in helloworld.proto
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.grpcPort))
+	if err != nil {
+		logger.Error("failed to listen on grpc port", zap.Int("port", s.grpcPort))
+		return err
+	}
+	grpcServer := grpc.NewServer()
+	defer grpcServer.GracefulStop()
+	helloworldproto.RegisterGreeterServer(grpcServer, &api{})
+	go func() {
+		if err := grpcServer.Serve(lis); err != nil {
+			logger.Error("failed to server", zap.Error(err))
+		}
+	}()
+
+	// Start a http server to handle http requests
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", httpHandler)
+	mux.HandleFunc("/account", httpHandler)
+	mux.HandleFunc("/account/profile", httpHandler)
+	mux.HandleFunc("/api/message", httpHandler)
+	hs := &http.Server{
+		Addr:    fmt.Sprintf(":%d", s.httpPort),
+		Handler: mux,
+	}
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		hs.Shutdown(ctx)
+	}()
+	go func() {
+		err := hs.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			logger.Error("failed to run metrics server", zap.Error(err))
+		}
+	}()
+
+	// Wait until we got a signal
+	<-ctx.Done()
+	return nil
+}
+
+func httpHandler(w http.ResponseWriter, req *http.Request) {
+	codes := []int{200, 200, 200, 200, 200, 200, 200, 200, 404, 500}
+	code := codes[rand.Int()%len(codes)]
+	response := fmt.Sprintf("reponse code: %d", code)
+
+	d := time.Duration((rand.Int()%5)+1) * 50 * time.Millisecond
+	time.Sleep(d)
+
+	if code == 200 {
+		fmt.Fprintln(w, response)
+		return
+	}
+	http.Error(w, response, code)
+}
+
+type api struct{}
+
+func (a *api) SayHello(ctx context.Context, in *helloworldproto.HelloRequest) (*helloworldproto.HelloResponse, error) {
+	time.Sleep(time.Duration(rand.Float64()) * time.Second)
+
+	if rand.Int()%200 == 0 {
+		return nil, status.Error(codes.Internal, "api: internal error")
+	}
+	return &helloworldproto.HelloResponse{
+		Message: "Hello " + in.Name,
+	}, nil
+}
+
+func (a *api) GetProfile(ctx context.Context, in *helloworldproto.ProfileRequest) (*helloworldproto.ProfileResponse, error) {
+	time.Sleep(time.Duration(rand.Float64()) * time.Second)
+
+	if rand.Int()%100 == 0 {
+		return nil, status.Error(codes.Internal, "api: internal error")
+	}
+	return &helloworldproto.ProfileResponse{
+		Name:     in.Name,
+		Homepage: fmt.Sprintf("http://helloworld/%s", in.Name),
+	}, nil
+}
diff --git a/pkg/app/example/cmd/simplegrpc/BUILD.bazel b/pkg/app/example/cmd/simplegrpc/BUILD.bazel
new file mode 100644
index 0000000..a69902f
--- /dev/null
+++ b/pkg/app/example/cmd/simplegrpc/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["scenario.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/cmd/simplegrpc",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/example/helloworld:go_default_library",
+        "//pkg/cli:go_default_library",
+        "//pkg/metrics:go_default_library",
+        "//pkg/metrics/grpcmetrics:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/example/cmd/simplegrpc/scenario.go b/pkg/app/example/cmd/simplegrpc/scenario.go
new file mode 100644
index 0000000..17d6502
--- /dev/null
+++ b/pkg/app/example/cmd/simplegrpc/scenario.go
@@ -0,0 +1,68 @@
+package simplegrpc
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+
+	helloworldproto "github.com/nghialv/lotus/pkg/app/example/helloworld"
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/metrics"
+	"github.com/nghialv/lotus/pkg/metrics/grpcmetrics"
+)
+
+type scenario struct {
+	helloWorldGRPCAddress string
+}
+
+func NewCommand() *cobra.Command {
+	s := &scenario{}
+	cmd := &cobra.Command{
+		Use:   "simple-grpc-scenario",
+		Short: "Start running simple grpc scenario",
+		RunE:  cli.WithContext(s.run),
+	}
+	cmd.Flags().StringVar(&s.helloWorldGRPCAddress, "helloworld-grpc-address", s.helloWorldGRPCAddress, "The grpc address to helloword service")
+	cmd.MarkFlagRequired("helloworld-grpc-address")
+	return cmd
+}
+
+func (s *scenario) run(ctx context.Context, logger *zap.Logger) error {
+	// Expose a metrics server
+	ms, err := metrics.NewServer(
+		8081,
+		metrics.WithLogger(logger.Sugar()),
+	)
+	if err != nil {
+		logger.Error("failed to create metrics server", zap.Error(err))
+		return err
+	}
+	defer ms.Stop()
+	go ms.Run()
+
+	// Just send a grpc rpc to helloworld server
+	conn, err := grpc.Dial(
+		s.helloWorldGRPCAddress,
+		grpc.WithStatsHandler(&grpcmetrics.ClientHandler{}),
+		grpc.WithInsecure(),
+	)
+	if err != nil {
+		logger.Error("failed to connect to helloworld server", zap.Error(err))
+		return err
+	}
+	defer conn.Close()
+
+	client := helloworldproto.NewGreeterClient(conn)
+	_, err = client.SayHello(ctx, &helloworldproto.HelloRequest{
+		Name: "lotus",
+	})
+	if err != nil {
+		logger.Error("failed to say hello to server", zap.Error(err))
+	}
+
+	// Wait until we got a signal
+	<-ctx.Done()
+	return nil
+}
diff --git a/pkg/app/example/cmd/simplehttp/BUILD.bazel b/pkg/app/example/cmd/simplehttp/BUILD.bazel
new file mode 100644
index 0000000..fa4b157
--- /dev/null
+++ b/pkg/app/example/cmd/simplehttp/BUILD.bazel
@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["scenario.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/cmd/simplehttp",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/cli:go_default_library",
+        "//pkg/metrics:go_default_library",
+        "//pkg/metrics/httpmetrics:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_golang_x_net//context/ctxhttp:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/example/cmd/simplehttp/scenario.go b/pkg/app/example/cmd/simplehttp/scenario.go
new file mode 100644
index 0000000..a76140d
--- /dev/null
+++ b/pkg/app/example/cmd/simplehttp/scenario.go
@@ -0,0 +1,57 @@
+package simplehttp
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"golang.org/x/net/context/ctxhttp"
+
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/metrics"
+	"github.com/nghialv/lotus/pkg/metrics/httpmetrics"
+)
+
+type scenario struct {
+}
+
+func NewCommand() *cobra.Command {
+	s := &scenario{}
+	cmd := &cobra.Command{
+		Use:   "simple-http-scenario",
+		Short: "Start running simple http scenario",
+		RunE:  cli.WithContext(s.run),
+	}
+	return cmd
+}
+
+func (s *scenario) run(ctx context.Context, logger *zap.Logger) error {
+	// Expose a metrics server
+	ms, err := metrics.NewServer(
+		8081,
+		metrics.WithLogger(logger.Sugar()),
+	)
+	if err != nil {
+		logger.Error("failed to create metrics server", zap.Error(err))
+		return err
+	}
+	defer ms.Stop()
+	go ms.Run()
+
+	// Just send a http request
+	client := &http.Client{
+		Transport: &httpmetrics.Transport{
+			UsePathAsRoute: true,
+		},
+	}
+	resp, err := ctxhttp.Get(ctx, client, "http://httpbin.org/user-agent")
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+
+	// Wait until we got a signal
+	<-ctx.Done()
+	return nil
+}
diff --git a/pkg/app/example/cmd/threesteps/BUILD.bazel b/pkg/app/example/cmd/threesteps/BUILD.bazel
new file mode 100644
index 0000000..d0c5ff8
--- /dev/null
+++ b/pkg/app/example/cmd/threesteps/BUILD.bazel
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["scenario.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/cmd/threesteps",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/example/helloworld:go_default_library",
+        "//pkg/cli:go_default_library",
+        "//pkg/metrics:go_default_library",
+        "//pkg/metrics/grpcmetrics:go_default_library",
+        "//pkg/metrics/httpmetrics:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_x_net//context/ctxhttp:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/example/cmd/threesteps/scenario.go b/pkg/app/example/cmd/threesteps/scenario.go
new file mode 100644
index 0000000..a080629
--- /dev/null
+++ b/pkg/app/example/cmd/threesteps/scenario.go
@@ -0,0 +1,162 @@
+package threesteps
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"golang.org/x/net/context/ctxhttp"
+	"google.golang.org/grpc"
+
+	helloworldproto "github.com/nghialv/lotus/pkg/app/example/helloworld"
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/metrics"
+	"github.com/nghialv/lotus/pkg/metrics/grpcmetrics"
+	"github.com/nghialv/lotus/pkg/metrics/httpmetrics"
+)
+
+var steps = []string{
+	"preparer",
+	"worker",
+	"cleaner",
+}
+
+type scenario struct {
+	step                  string
+	duration              time.Duration
+	helloWorldGRPCAddress string
+	helloWorldHTTPAddress string
+
+	httpClient *http.Client
+	grpcClient helloworldproto.GreeterClient
+}
+
+func NewCommand() *cobra.Command {
+	s := &scenario{
+		step:     steps[0],
+		duration: 10 * time.Second,
+	}
+	cmd := &cobra.Command{
+		Use:   "three-steps-scenario",
+		Short: "Start running three steps scenario",
+		RunE:  cli.WithContext(s.run),
+	}
+	cmd.Flags().StringVar(&s.step, "step", s.step, "The step will be run")
+	cmd.Flags().DurationVar(&s.duration, "duration", s.duration, "How long this step should be run (Only valid preparer or cleaner)")
+	cmd.Flags().StringVar(&s.helloWorldGRPCAddress, "helloworld-grpc-address", s.helloWorldGRPCAddress, "The grpc address to helloword service")
+	cmd.MarkFlagRequired("helloworld-grpc-address")
+	cmd.Flags().StringVar(&s.helloWorldHTTPAddress, "helloworld-http-address", s.helloWorldHTTPAddress, "The http address to helloword service")
+	cmd.MarkFlagRequired("helloworld-http-address")
+	return cmd
+}
+
+func (s *scenario) run(ctx context.Context, logger *zap.Logger) error {
+	// Expose a metrics server
+	ms, err := metrics.NewServer(
+		8081,
+		metrics.WithLogger(logger.Sugar()),
+	)
+	if err != nil {
+		logger.Error("failed to create metrics server", zap.Error(err))
+		return err
+	}
+	defer ms.Stop()
+	go ms.Run()
+
+	// For worker step
+	if s.step == "worker" {
+		return s.runWorker(ctx, logger)
+	}
+
+	// For preparer or cleaner step we just wait
+	select {
+	case <-time.After(s.duration):
+		return nil
+	case <-ctx.Done():
+		return nil
+	}
+}
+
+func (s *scenario) runWorker(ctx context.Context, logger *zap.Logger) error {
+	s.httpClient = &http.Client{
+		Transport: &httpmetrics.Transport{
+			UsePathAsRoute: true,
+		},
+	}
+	conn, err := grpc.Dial(
+		s.helloWorldGRPCAddress,
+		grpc.WithStatsHandler(&grpcmetrics.ClientHandler{}),
+		grpc.WithInsecure(),
+	)
+	if err != nil {
+		logger.Error("failed to connect to helloworl server", zap.Error(err))
+		return err
+	}
+	defer conn.Close()
+	s.grpcClient = helloworldproto.NewGreeterClient(conn)
+
+	// Periodically send the requests
+	ticker := time.NewTicker(200 * time.Millisecond)
+	for {
+		select {
+		case <-ticker.C:
+			x := rand.Int() % 5
+			if x == 0 {
+				s.sendHTTP(ctx, logger)
+				break
+			}
+			s.sendGRPC(ctx, x+1, logger)
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+func (s *scenario) sendHTTP(ctx context.Context, logger *zap.Logger) {
+	paths := []string{
+		"/",
+		"/account",
+		"/account/profile",
+		"/api/message",
+	}
+	x := rand.Int() % len(paths)
+	address := fmt.Sprintf("%s%s", s.helloWorldHTTPAddress, paths[x])
+
+	//ctx, _ = metrics.ContextWithRoute(ctx, "foo")
+	resp, err := ctxhttp.Get(ctx, s.httpClient, address)
+	if err != nil {
+		return
+	}
+	io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+
+	b := strings.NewReader(`{"key":"value"}`)
+	resp, err = ctxhttp.Post(ctx, s.httpClient, address, "application/json", b)
+	if err != nil {
+		return
+	}
+	//io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+}
+
+func (s *scenario) sendGRPC(ctx context.Context, rpcs int, logger *zap.Logger) {
+	for i := 0; i < rpcs; i++ {
+		name := fmt.Sprintf("name-%d", i)
+		var err error
+		if i%2 == 0 {
+			_, err = s.grpcClient.SayHello(ctx, &helloworldproto.HelloRequest{Name: name})
+		} else {
+			_, err = s.grpcClient.GetProfile(ctx, &helloworldproto.ProfileRequest{Name: name})
+		}
+		if err != nil {
+			logger.Warn("failed to send grpc rpc", zap.Error(err))
+		}
+	}
+}
diff --git a/pkg/app/example/cmd/virtualuser/BUILD.bazel b/pkg/app/example/cmd/virtualuser/BUILD.bazel
new file mode 100644
index 0000000..ab99b0c
--- /dev/null
+++ b/pkg/app/example/cmd/virtualuser/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "scenario.go",
+        "user.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/cmd/virtualuser",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/example/helloworld:go_default_library",
+        "//pkg/cli:go_default_library",
+        "//pkg/metrics:go_default_library",
+        "//pkg/metrics/grpcmetrics:go_default_library",
+        "//pkg/virtualuser:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+        "@org_golang_google_grpc//status:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/example/cmd/virtualuser/scenario.go b/pkg/app/example/cmd/virtualuser/scenario.go
new file mode 100644
index 0000000..caf300f
--- /dev/null
+++ b/pkg/app/example/cmd/virtualuser/scenario.go
@@ -0,0 +1,65 @@
+package virtualuser
+
+import (
+	"context"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/cli"
+	"github.com/nghialv/lotus/pkg/metrics"
+	"github.com/nghialv/lotus/pkg/virtualuser"
+)
+
+type scenario struct {
+	hatchRate             int
+	numVirtualUsers       int
+	helloWorldGRPCAddress string
+}
+
+func NewCommand() *cobra.Command {
+	s := &scenario{
+		hatchRate:       1,
+		numVirtualUsers: 10,
+	}
+	cmd := &cobra.Command{
+		Use:   "virtual-user-scenario",
+		Short: "Start running virtual user scenario",
+		RunE:  cli.WithContext(s.run),
+	}
+	cmd.Flags().IntVar(&s.hatchRate, "hatch-rate", s.hatchRate, "How many virtual users should be spwawn per second")
+	cmd.Flags().IntVar(&s.numVirtualUsers, "num-virtual-users", s.numVirtualUsers, "How many virtual users should be spawn in total")
+	cmd.Flags().StringVar(&s.helloWorldGRPCAddress, "helloworld-grpc-address", s.helloWorldGRPCAddress, "The grpc address to helloword service")
+	cmd.MarkFlagRequired("helloworld-grpc-address")
+	return cmd
+}
+
+func (s *scenario) run(ctx context.Context, logger *zap.Logger) error {
+	// Expose a metrics server
+	ms, err := metrics.NewServer(
+		8081,
+		metrics.WithLogger(logger.Sugar()),
+	)
+	if err != nil {
+		logger.Error("failed to create metrics server", zap.Error(err))
+		return err
+	}
+	defer ms.Stop()
+	go ms.Run()
+
+	// Start a group of virtual users
+	group := virtualuser.NewGroup(
+		s.numVirtualUsers,
+		s.hatchRate,
+		func() (virtualuser.VirtualUser, error) {
+			return newUser(s.helloWorldGRPCAddress, logger)
+		},
+	)
+	defer group.Stop(time.Second)
+	go group.Run(ctx)
+
+	// Wait until we got a signal
+	<-ctx.Done()
+	return nil
+}
diff --git a/pkg/app/example/cmd/virtualuser/user.go b/pkg/app/example/cmd/virtualuser/user.go
new file mode 100644
index 0000000..bef64a8
--- /dev/null
+++ b/pkg/app/example/cmd/virtualuser/user.go
@@ -0,0 +1,65 @@
+package virtualuser
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	helloworldproto "github.com/nghialv/lotus/pkg/app/example/helloworld"
+	"github.com/nghialv/lotus/pkg/metrics/grpcmetrics"
+)
+
+type User struct {
+	conn   *grpc.ClientConn
+	client helloworldproto.GreeterClient
+	logger *zap.Logger
+}
+
+func newUser(address string, logger *zap.Logger) (*User, error) {
+	conn, err := grpc.Dial(
+		address,
+		grpc.WithStatsHandler(&grpcmetrics.ClientHandler{}),
+		grpc.WithInsecure(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return &User{
+		conn:   conn,
+		client: helloworldproto.NewGreeterClient(conn),
+		logger: logger,
+	}, nil
+}
+
+func (u *User) Run(ctx context.Context) error {
+	defer u.conn.Close()
+	var index int = 0
+	ticker := time.NewTicker(500 * time.Millisecond)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			u.sendGRPC(ctx, index)
+			index++
+		}
+	}
+}
+
+func (u *User) sendGRPC(ctx context.Context, index int) error {
+	name := fmt.Sprintf("name-%d", index)
+	_, err := u.client.SayHello(ctx, &helloworldproto.HelloRequest{Name: name})
+	if err != nil {
+		code := status.Code(err)
+		if code != codes.Canceled {
+			u.logger.Warn("failed to send grpc rpc", zap.Error(err))
+			return err
+		}
+	}
+	return nil
+}
diff --git a/pkg/app/example/helloworld/BUILD.bazel b/pkg/app/example/helloworld/BUILD.bazel
new file mode 100644
index 0000000..94057d8
--- /dev/null
+++ b/pkg/app/example/helloworld/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+proto_library(
+    name = "helloworld_proto",
+    srcs = ["helloworld.proto"],
+    visibility = ["//visibility:public"],
+)
+
+go_proto_library(
+    name = "helloworld_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:go_grpc"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/helloworld",
+    proto = ":helloworld_proto",
+    visibility = ["//visibility:public"],
+)
+
+go_library(
+    name = "go_default_library",
+    embed = [":helloworld_go_proto"],
+    importpath = "github.com/nghialv/lotus/pkg/app/example/helloworld",
+    visibility = ["//visibility:public"],
+)
diff --git a/pkg/app/example/helloworld/helloworld.proto b/pkg/app/example/helloworld/helloworld.proto
new file mode 100644
index 0000000..8e130ae
--- /dev/null
+++ b/pkg/app/example/helloworld/helloworld.proto
@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package pkg.example.helloworld;
+option go_package = "helloworld";
+
+service Greeter {
+  rpc SayHello (HelloRequest) returns (HelloResponse) {}
+  rpc GetProfile (ProfileRequest) returns (ProfileResponse) {}
+}
+
+message HelloRequest {
+  string name = 1;
+}
+
+message HelloResponse {
+  string message = 1;
+}
+
+message ProfileRequest {
+  string name = 1;
+}
+
+message ProfileResponse{
+  string name = 1;
+  string homepage = 2;
+}
diff --git a/pkg/app/lotus/apis/lotus/BUILD.bazel b/pkg/app/lotus/apis/lotus/BUILD.bazel
new file mode 100644
index 0000000..6dc85d3
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["register.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus",
+    visibility = ["//visibility:public"],
+)
diff --git a/pkg/app/lotus/apis/lotus/register.go b/pkg/app/lotus/apis/lotus/register.go
new file mode 100644
index 0000000..047719f
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/register.go
@@ -0,0 +1,5 @@
+package lotus
+
+const (
+	GroupName = "lotus.nghialv.com"
+)
diff --git a/pkg/app/lotus/apis/lotus/v1beta1/BUILD.bazel b/pkg/app/lotus/apis/lotus/v1beta1/BUILD.bazel
new file mode 100644
index 0000000..55064c7
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/v1beta1/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "register.go",
+        "types.go",
+        "zz_generated.deepcopy.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus:go_default_library",
+        "@io_k8s_api//core/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/apis/lotus/v1beta1/doc.go b/pkg/app/lotus/apis/lotus/v1beta1/doc.go
new file mode 100644
index 0000000..b9fc395
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/v1beta1/doc.go
@@ -0,0 +1,5 @@
+// +k8s:deepcopy-gen=package
+// +groupName=lotus.nghialv.com
+
+// Package v1beta1 is the v1beta1 version of the API.
+package v1beta1
diff --git a/pkg/app/lotus/apis/lotus/v1beta1/register.go b/pkg/app/lotus/apis/lotus/v1beta1/register.go
new file mode 100644
index 0000000..5b543ce
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/v1beta1/register.go
@@ -0,0 +1,37 @@
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	lotus "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus"
+)
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: lotus.GroupName, Version: "v1beta1"}
+
+// Kind takes an unqualified kind and returns back a Group qualified GroupKind
+func Kind(kind string) schema.GroupKind {
+	return SchemeGroupVersion.WithKind(kind).GroupKind()
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = SchemeBuilder.AddToScheme
+)
+
+// Adds the list of known types to Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&Lotus{},
+		&LotusList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}
diff --git a/pkg/app/lotus/apis/lotus/v1beta1/types.go b/pkg/app/lotus/apis/lotus/v1beta1/types.go
new file mode 100644
index 0000000..241244d
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/v1beta1/types.go
@@ -0,0 +1,85 @@
+package v1beta1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type Lotus struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   LotusSpec   `json:"spec"`
+	Status LotusStatus `json:"status"`
+}
+
+type LotusSpec struct {
+	TTLSecondsAfterFinished  *int32 `json:"ttlSecondsAfterFinished"`
+	CheckIntervalSeconds     *int32 `json:"checkIntervalSeconds"`
+	CheckInitialDelaySeconds *int32 `json:"checkInitialDelaySeconds"`
+
+	Preparer *LotusSpecPreparer `json:"preparer"`
+	Worker   *LotusSpecWorker   `json:"worker"`
+	Cleaner  *LotusSpecCleaner  `json:"cleaner"`
+	Checks   []LotusCheck       `json:"checks"`
+}
+
+type LotusSpecWorker struct {
+	RunTime     string             `json:"runTime"`
+	Replicas    *int32             `json:"replicas"`
+	MetricsPort *int32             `json:"metricsPort"`
+	Containers  []corev1.Container `json:"containers"`
+	Volumes     []corev1.Volume    `json:"volumes"`
+}
+
+type LotusSpecPreparer struct {
+	Containers []corev1.Container `json:"containers"`
+	Volumes    []corev1.Volume    `json:"volumes"`
+}
+
+type LotusSpecCleaner struct {
+	Containers []corev1.Container `json:"containers"`
+	Volumes    []corev1.Volume    `json:"volumes"`
+}
+
+type LotusCheck struct {
+	Name       string `json:"name"`
+	Expr       string `json:"expr"`
+	For        string `json:"for"`
+	DataSource string `json:"dataSource"`
+}
+
+type LotusPhase string
+
+const (
+	LotusInit            LotusPhase = ""
+	LotusPending                    = "Pending"
+	LotusPreparing                  = "Preparing"
+	LotusRunning                    = "Running"
+	LotusCleaning                   = "Cleaning"
+	LotusFailureCleaning            = "FailureCleaning"
+	LotusSucceeded                  = "Succeeded"
+	LotusFailed                     = "Failed"
+)
+
+type LotusStatus struct {
+	PreparerStartTime      *metav1.Time `json:"preparerStartTime"`
+	PreparerCompletionTime *metav1.Time `json:"preparerCompletionTime"`
+	WorkerStartTime        *metav1.Time `json:"workerStartTime"`
+	WorkerCompletionTime   *metav1.Time `json:"workerCompletionTime"`
+	CleanerStartTime       *metav1.Time `json:"cleanerStartTime"`
+	CleanerCompletionTime  *metav1.Time `json:"cleanerCompletionTime"`
+	Phase                  LotusPhase   `json:"phase"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type LotusList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []Lotus `json:"items"`
+}
diff --git a/pkg/app/lotus/apis/lotus/v1beta1/zz_generated.deepcopy.go b/pkg/app/lotus/apis/lotus/v1beta1/zz_generated.deepcopy.go
new file mode 100644
index 0000000..3620944
--- /dev/null
+++ b/pkg/app/lotus/apis/lotus/v1beta1/zz_generated.deepcopy.go
@@ -0,0 +1,284 @@
+// +build !ignore_autogenerated
+
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Lotus) DeepCopyInto(out *Lotus) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lotus.
+func (in *Lotus) DeepCopy() *Lotus {
+	if in == nil {
+		return nil
+	}
+	out := new(Lotus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *Lotus) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusCheck) DeepCopyInto(out *LotusCheck) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusCheck.
+func (in *LotusCheck) DeepCopy() *LotusCheck {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusCheck)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusList) DeepCopyInto(out *LotusList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	out.ListMeta = in.ListMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]Lotus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusList.
+func (in *LotusList) DeepCopy() *LotusList {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *LotusList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusSpec) DeepCopyInto(out *LotusSpec) {
+	*out = *in
+	if in.TTLSecondsAfterFinished != nil {
+		in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished
+		*out = new(int32)
+		**out = **in
+	}
+	if in.CheckIntervalSeconds != nil {
+		in, out := &in.CheckIntervalSeconds, &out.CheckIntervalSeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.CheckInitialDelaySeconds != nil {
+		in, out := &in.CheckInitialDelaySeconds, &out.CheckInitialDelaySeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Preparer != nil {
+		in, out := &in.Preparer, &out.Preparer
+		*out = new(LotusSpecPreparer)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Worker != nil {
+		in, out := &in.Worker, &out.Worker
+		*out = new(LotusSpecWorker)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Cleaner != nil {
+		in, out := &in.Cleaner, &out.Cleaner
+		*out = new(LotusSpecCleaner)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Checks != nil {
+		in, out := &in.Checks, &out.Checks
+		*out = make([]LotusCheck, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusSpec.
+func (in *LotusSpec) DeepCopy() *LotusSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusSpecCleaner) DeepCopyInto(out *LotusSpecCleaner) {
+	*out = *in
+	if in.Containers != nil {
+		in, out := &in.Containers, &out.Containers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusSpecCleaner.
+func (in *LotusSpecCleaner) DeepCopy() *LotusSpecCleaner {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusSpecCleaner)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusSpecPreparer) DeepCopyInto(out *LotusSpecPreparer) {
+	*out = *in
+	if in.Containers != nil {
+		in, out := &in.Containers, &out.Containers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusSpecPreparer.
+func (in *LotusSpecPreparer) DeepCopy() *LotusSpecPreparer {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusSpecPreparer)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusSpecWorker) DeepCopyInto(out *LotusSpecWorker) {
+	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MetricsPort != nil {
+		in, out := &in.MetricsPort, &out.MetricsPort
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Containers != nil {
+		in, out := &in.Containers, &out.Containers
+		*out = make([]v1.Container, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Volumes != nil {
+		in, out := &in.Volumes, &out.Volumes
+		*out = make([]v1.Volume, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusSpecWorker.
+func (in *LotusSpecWorker) DeepCopy() *LotusSpecWorker {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusSpecWorker)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LotusStatus) DeepCopyInto(out *LotusStatus) {
+	*out = *in
+	if in.PreparerStartTime != nil {
+		in, out := &in.PreparerStartTime, &out.PreparerStartTime
+		*out = (*in).DeepCopy()
+	}
+	if in.PreparerCompletionTime != nil {
+		in, out := &in.PreparerCompletionTime, &out.PreparerCompletionTime
+		*out = (*in).DeepCopy()
+	}
+	if in.WorkerStartTime != nil {
+		in, out := &in.WorkerStartTime, &out.WorkerStartTime
+		*out = (*in).DeepCopy()
+	}
+	if in.WorkerCompletionTime != nil {
+		in, out := &in.WorkerCompletionTime, &out.WorkerCompletionTime
+		*out = (*in).DeepCopy()
+	}
+	if in.CleanerStartTime != nil {
+		in, out := &in.CleanerStartTime, &out.CleanerStartTime
+		*out = (*in).DeepCopy()
+	}
+	if in.CleanerCompletionTime != nil {
+		in, out := &in.CleanerCompletionTime, &out.CleanerCompletionTime
+		*out = (*in).DeepCopy()
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LotusStatus.
+func (in *LotusStatus) DeepCopy() *LotusStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(LotusStatus)
+	in.DeepCopyInto(out)
+	return out
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/BUILD.bazel b/pkg/app/lotus/client/clientset/versioned/BUILD.bazel
new file mode 100644
index 0000000..b48179a
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clientset.go",
+        "doc.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1:go_default_library",
+        "@io_k8s_client_go//discovery:go_default_library",
+        "@io_k8s_client_go//rest:go_default_library",
+        "@io_k8s_client_go//util/flowcontrol:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/clientset/versioned/clientset.go b/pkg/app/lotus/client/clientset/versioned/clientset.go
new file mode 100644
index 0000000..468534f
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/clientset.go
@@ -0,0 +1,88 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package versioned
+
+import (
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1"
+	discovery "k8s.io/client-go/discovery"
+	rest "k8s.io/client-go/rest"
+	flowcontrol "k8s.io/client-go/util/flowcontrol"
+)
+
+type Interface interface {
+	Discovery() discovery.DiscoveryInterface
+	LotusV1beta1() lotusv1beta1.LotusV1beta1Interface
+	// Deprecated: please explicitly pick a version if possible.
+	Lotus() lotusv1beta1.LotusV1beta1Interface
+}
+
+// Clientset contains the clients for groups. Each group has exactly one
+// version included in a Clientset.
+type Clientset struct {
+	*discovery.DiscoveryClient
+	lotusV1beta1 *lotusv1beta1.LotusV1beta1Client
+}
+
+// LotusV1beta1 retrieves the LotusV1beta1Client
+func (c *Clientset) LotusV1beta1() lotusv1beta1.LotusV1beta1Interface {
+	return c.lotusV1beta1
+}
+
+// Deprecated: Lotus retrieves the default version of LotusClient.
+// Please explicitly pick a version.
+func (c *Clientset) Lotus() lotusv1beta1.LotusV1beta1Interface {
+	return c.lotusV1beta1
+}
+
+// Discovery retrieves the DiscoveryClient
+func (c *Clientset) Discovery() discovery.DiscoveryInterface {
+	if c == nil {
+		return nil
+	}
+	return c.DiscoveryClient
+}
+
+// NewForConfig creates a new Clientset for the given config.
+func NewForConfig(c *rest.Config) (*Clientset, error) {
+	configShallowCopy := *c
+	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
+		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
+	}
+	var cs Clientset
+	var err error
+	cs.lotusV1beta1, err = lotusv1beta1.NewForConfig(&configShallowCopy)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
+	if err != nil {
+		return nil, err
+	}
+	return &cs, nil
+}
+
+// NewForConfigOrDie creates a new Clientset for the given config and
+// panics if there is an error in the config.
+func NewForConfigOrDie(c *rest.Config) *Clientset {
+	var cs Clientset
+	cs.lotusV1beta1 = lotusv1beta1.NewForConfigOrDie(c)
+
+	cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)
+	return &cs
+}
+
+// New creates a new Clientset for the given RESTClient.
+func New(c rest.Interface) *Clientset {
+	var cs Clientset
+	cs.lotusV1beta1 = lotusv1beta1.New(c)
+
+	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
+	return &cs
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/doc.go b/pkg/app/lotus/client/clientset/versioned/doc.go
new file mode 100644
index 0000000..7410ea1
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/doc.go
@@ -0,0 +1,10 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package has the automatically generated clientset.
+package versioned
diff --git a/pkg/app/lotus/client/clientset/versioned/fake/BUILD.bazel b/pkg/app/lotus/client/clientset/versioned/fake/BUILD.bazel
new file mode 100644
index 0000000..2de453a
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/fake/BUILD.bazel
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "clientset_generated.go",
+        "doc.go",
+        "register.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/fake",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/serializer:go_default_library",
+        "@io_k8s_apimachinery//pkg/watch:go_default_library",
+        "@io_k8s_client_go//discovery:go_default_library",
+        "@io_k8s_client_go//discovery/fake:go_default_library",
+        "@io_k8s_client_go//testing:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/clientset/versioned/fake/clientset_generated.go b/pkg/app/lotus/client/clientset/versioned/fake/clientset_generated.go
new file mode 100644
index 0000000..4bd3777
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/fake/clientset_generated.go
@@ -0,0 +1,72 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	clientset "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1"
+	fakelotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/discovery"
+	fakediscovery "k8s.io/client-go/discovery/fake"
+	"k8s.io/client-go/testing"
+)
+
+// NewSimpleClientset returns a clientset that will respond with the provided objects.
+// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
+// without applying any validations and/or defaults. It shouldn't be considered a replacement
+// for a real clientset and is mostly useful in simple unit tests.
+func NewSimpleClientset(objects ...runtime.Object) *Clientset {
+	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
+	for _, obj := range objects {
+		if err := o.Add(obj); err != nil {
+			panic(err)
+		}
+	}
+
+	cs := &Clientset{}
+	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
+	cs.AddReactor("*", "*", testing.ObjectReaction(o))
+	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
+		gvr := action.GetResource()
+		ns := action.GetNamespace()
+		watch, err := o.Watch(gvr, ns)
+		if err != nil {
+			return false, nil, err
+		}
+		return true, watch, nil
+	})
+
+	return cs
+}
+
+// Clientset implements clientset.Interface. Meant to be embedded into a
+// struct to get a default implementation. This makes faking out just the method
+// you want to test easier.
+type Clientset struct {
+	testing.Fake
+	discovery *fakediscovery.FakeDiscovery
+}
+
+func (c *Clientset) Discovery() discovery.DiscoveryInterface {
+	return c.discovery
+}
+
+var _ clientset.Interface = &Clientset{}
+
+// LotusV1beta1 retrieves the LotusV1beta1Client
+func (c *Clientset) LotusV1beta1() lotusv1beta1.LotusV1beta1Interface {
+	return &fakelotusv1beta1.FakeLotusV1beta1{Fake: &c.Fake}
+}
+
+// Lotus retrieves the LotusV1beta1Client
+func (c *Clientset) Lotus() lotusv1beta1.LotusV1beta1Interface {
+	return &fakelotusv1beta1.FakeLotusV1beta1{Fake: &c.Fake}
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/fake/doc.go b/pkg/app/lotus/client/clientset/versioned/fake/doc.go
new file mode 100644
index 0000000..2fb358b
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/fake/doc.go
@@ -0,0 +1,10 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package has the automatically generated fake clientset.
+package fake
diff --git a/pkg/app/lotus/client/clientset/versioned/fake/register.go b/pkg/app/lotus/client/clientset/versioned/fake/register.go
new file mode 100644
index 0000000..6018067
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/fake/register.go
@@ -0,0 +1,44 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var scheme = runtime.NewScheme()
+var codecs = serializer.NewCodecFactory(scheme)
+var parameterCodec = runtime.NewParameterCodec(scheme)
+
+func init() {
+	v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
+	AddToScheme(scheme)
+}
+
+// AddToScheme adds all types of this clientset into the given scheme. This allows composition
+// of clientsets, like in:
+//
+//   import (
+//     "k8s.io/client-go/kubernetes"
+//     clientsetscheme "k8s.io/client-go/kubernetes/scheme"
+//     aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
+//   )
+//
+//   kclientset, _ := kubernetes.NewForConfig(c)
+//   aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
+//
+// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
+// correctly.
+func AddToScheme(scheme *runtime.Scheme) {
+	lotusv1beta1.AddToScheme(scheme)
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/scheme/BUILD.bazel b/pkg/app/lotus/client/clientset/versioned/scheme/BUILD.bazel
new file mode 100644
index 0000000..80d6041
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/scheme/BUILD.bazel
@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "register.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/scheme",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/serializer:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/clientset/versioned/scheme/doc.go b/pkg/app/lotus/client/clientset/versioned/scheme/doc.go
new file mode 100644
index 0000000..6d289f2
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/scheme/doc.go
@@ -0,0 +1,10 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package contains the scheme of the automatically generated clientset.
+package scheme
diff --git a/pkg/app/lotus/client/clientset/versioned/scheme/register.go b/pkg/app/lotus/client/clientset/versioned/scheme/register.go
new file mode 100644
index 0000000..dabcb11
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/scheme/register.go
@@ -0,0 +1,44 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package scheme
+
+import (
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var Scheme = runtime.NewScheme()
+var Codecs = serializer.NewCodecFactory(Scheme)
+var ParameterCodec = runtime.NewParameterCodec(Scheme)
+
+func init() {
+	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
+	AddToScheme(Scheme)
+}
+
+// AddToScheme adds all types of this clientset into the given scheme. This allows composition
+// of clientsets, like in:
+//
+//   import (
+//     "k8s.io/client-go/kubernetes"
+//     clientsetscheme "k8s.io/client-go/kubernetes/scheme"
+//     aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
+//   )
+//
+//   kclientset, _ := kubernetes.NewForConfig(c)
+//   aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
+//
+// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
+// correctly.
+func AddToScheme(scheme *runtime.Scheme) {
+	lotusv1beta1.AddToScheme(scheme)
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/BUILD.bazel b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/BUILD.bazel
new file mode 100644
index 0000000..651b537
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/BUILD.bazel
@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "generated_expansion.go",
+        "lotus.go",
+        "lotus_client.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned/scheme:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/serializer:go_default_library",
+        "@io_k8s_apimachinery//pkg/types:go_default_library",
+        "@io_k8s_apimachinery//pkg/watch:go_default_library",
+        "@io_k8s_client_go//rest:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/doc.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/doc.go
new file mode 100644
index 0000000..2229cd1
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/doc.go
@@ -0,0 +1,10 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// This package has the automatically generated typed clients.
+package v1beta1
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/BUILD.bazel b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/BUILD.bazel
new file mode 100644
index 0000000..2abc1c2
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "fake_lotus.go",
+        "fake_lotus_client.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/labels:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_apimachinery//pkg/types:go_default_library",
+        "@io_k8s_apimachinery//pkg/watch:go_default_library",
+        "@io_k8s_client_go//rest:go_default_library",
+        "@io_k8s_client_go//testing:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/doc.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/doc.go
new file mode 100644
index 0000000..d7f30ba
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/doc.go
@@ -0,0 +1,10 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+// Package fake has the automatically generated clients.
+package fake
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus.go
new file mode 100644
index 0000000..75f01ba
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus.go
@@ -0,0 +1,130 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+)
+
+// FakeLotuses implements LotusInterface
+type FakeLotuses struct {
+	Fake *FakeLotusV1beta1
+	ns   string
+}
+
+var lotusesResource = schema.GroupVersionResource{Group: "lotus.nghialv.com", Version: "v1beta1", Resource: "lotuses"}
+
+var lotusesKind = schema.GroupVersionKind{Group: "lotus.nghialv.com", Version: "v1beta1", Kind: "Lotus"}
+
+// Get takes name of the lotus, and returns the corresponding lotus object, and an error if there is any.
+func (c *FakeLotuses) Get(name string, options v1.GetOptions) (result *v1beta1.Lotus, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewGetAction(lotusesResource, c.ns, name), &v1beta1.Lotus{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.Lotus), err
+}
+
+// List takes label and field selectors, and returns the list of Lotuses that match those selectors.
+func (c *FakeLotuses) List(opts v1.ListOptions) (result *v1beta1.LotusList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewListAction(lotusesResource, lotusesKind, c.ns, opts), &v1beta1.LotusList{})
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1beta1.LotusList{ListMeta: obj.(*v1beta1.LotusList).ListMeta}
+	for _, item := range obj.(*v1beta1.LotusList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested lotuses.
+func (c *FakeLotuses) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewWatchAction(lotusesResource, c.ns, opts))
+
+}
+
+// Create takes the representation of a lotus and creates it.  Returns the server's representation of the lotus, and an error, if there is any.
+func (c *FakeLotuses) Create(lotus *v1beta1.Lotus) (result *v1beta1.Lotus, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewCreateAction(lotusesResource, c.ns, lotus), &v1beta1.Lotus{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.Lotus), err
+}
+
+// Update takes the representation of a lotus and updates it. Returns the server's representation of the lotus, and an error, if there is any.
+func (c *FakeLotuses) Update(lotus *v1beta1.Lotus) (result *v1beta1.Lotus, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateAction(lotusesResource, c.ns, lotus), &v1beta1.Lotus{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.Lotus), err
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *FakeLotuses) UpdateStatus(lotus *v1beta1.Lotus) (*v1beta1.Lotus, error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateSubresourceAction(lotusesResource, "status", c.ns, lotus), &v1beta1.Lotus{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.Lotus), err
+}
+
+// Delete takes name of the lotus and deletes it. Returns an error if one occurs.
+func (c *FakeLotuses) Delete(name string, options *v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewDeleteAction(lotusesResource, c.ns, name), &v1beta1.Lotus{})
+
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeLotuses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	action := testing.NewDeleteCollectionAction(lotusesResource, c.ns, listOptions)
+
+	_, err := c.Fake.Invokes(action, &v1beta1.LotusList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched lotus.
+func (c *FakeLotuses) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Lotus, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewPatchSubresourceAction(lotusesResource, c.ns, name, data, subresources...), &v1beta1.Lotus{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.Lotus), err
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus_client.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus_client.go
new file mode 100644
index 0000000..9035a02
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/fake/fake_lotus_client.go
@@ -0,0 +1,30 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1"
+	rest "k8s.io/client-go/rest"
+	testing "k8s.io/client-go/testing"
+)
+
+type FakeLotusV1beta1 struct {
+	*testing.Fake
+}
+
+func (c *FakeLotusV1beta1) Lotuses(namespace string) v1beta1.LotusInterface {
+	return &FakeLotuses{c, namespace}
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *FakeLotusV1beta1) RESTClient() rest.Interface {
+	var ret *rest.RESTClient
+	return ret
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/generated_expansion.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/generated_expansion.go
new file mode 100644
index 0000000..588037e
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/generated_expansion.go
@@ -0,0 +1,11 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1beta1
+
+type LotusExpansion interface{}
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus.go
new file mode 100644
index 0000000..1b012bb
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus.go
@@ -0,0 +1,164 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	scheme "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/scheme"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+)
+
+// LotusesGetter has a method to return a LotusInterface.
+// A group's client should implement this interface.
+type LotusesGetter interface {
+	Lotuses(namespace string) LotusInterface
+}
+
+// LotusInterface has methods to work with Lotus resources.
+type LotusInterface interface {
+	Create(*v1beta1.Lotus) (*v1beta1.Lotus, error)
+	Update(*v1beta1.Lotus) (*v1beta1.Lotus, error)
+	UpdateStatus(*v1beta1.Lotus) (*v1beta1.Lotus, error)
+	Delete(name string, options *v1.DeleteOptions) error
+	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
+	Get(name string, options v1.GetOptions) (*v1beta1.Lotus, error)
+	List(opts v1.ListOptions) (*v1beta1.LotusList, error)
+	Watch(opts v1.ListOptions) (watch.Interface, error)
+	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Lotus, err error)
+	LotusExpansion
+}
+
+// lotuses implements LotusInterface
+type lotuses struct {
+	client rest.Interface
+	ns     string
+}
+
+// newLotuses returns a Lotuses
+func newLotuses(c *LotusV1beta1Client, namespace string) *lotuses {
+	return &lotuses{
+		client: c.RESTClient(),
+		ns:     namespace,
+	}
+}
+
+// Get takes name of the lotus, and returns the corresponding lotus object, and an error if there is any.
+func (c *lotuses) Get(name string, options v1.GetOptions) (result *v1beta1.Lotus, err error) {
+	result = &v1beta1.Lotus{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("lotuses").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do().
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of Lotuses that match those selectors.
+func (c *lotuses) List(opts v1.ListOptions) (result *v1beta1.LotusList, err error) {
+	result = &v1beta1.LotusList{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("lotuses").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested lotuses.
+func (c *lotuses) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	opts.Watch = true
+	return c.client.Get().
+		Namespace(c.ns).
+		Resource("lotuses").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}
+
+// Create takes the representation of a lotus and creates it.  Returns the server's representation of the lotus, and an error, if there is any.
+func (c *lotuses) Create(lotus *v1beta1.Lotus) (result *v1beta1.Lotus, err error) {
+	result = &v1beta1.Lotus{}
+	err = c.client.Post().
+		Namespace(c.ns).
+		Resource("lotuses").
+		Body(lotus).
+		Do().
+		Into(result)
+	return
+}
+
+// Update takes the representation of a lotus and updates it. Returns the server's representation of the lotus, and an error, if there is any.
+func (c *lotuses) Update(lotus *v1beta1.Lotus) (result *v1beta1.Lotus, err error) {
+	result = &v1beta1.Lotus{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("lotuses").
+		Name(lotus.Name).
+		Body(lotus).
+		Do().
+		Into(result)
+	return
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+
+func (c *lotuses) UpdateStatus(lotus *v1beta1.Lotus) (result *v1beta1.Lotus, err error) {
+	result = &v1beta1.Lotus{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("lotuses").
+		Name(lotus.Name).
+		SubResource("status").
+		Body(lotus).
+		Do().
+		Into(result)
+	return
+}
+
+// Delete takes name of the lotus and deletes it. Returns an error if one occurs.
+func (c *lotuses) Delete(name string, options *v1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("lotuses").
+		Name(name).
+		Body(options).
+		Do().
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *lotuses) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("lotuses").
+		VersionedParams(&listOptions, scheme.ParameterCodec).
+		Body(options).
+		Do().
+		Error()
+}
+
+// Patch applies the patch and returns the patched lotus.
+func (c *lotuses) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.Lotus, err error) {
+	result = &v1beta1.Lotus{}
+	err = c.client.Patch(pt).
+		Namespace(c.ns).
+		Resource("lotuses").
+		SubResource(subresources...).
+		Name(name).
+		Body(data).
+		Do().
+		Into(result)
+	return
+}
diff --git a/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus_client.go b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus_client.go
new file mode 100644
index 0000000..04cd1fd
--- /dev/null
+++ b/pkg/app/lotus/client/clientset/versioned/typed/lotus/v1beta1/lotus_client.go
@@ -0,0 +1,80 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/scheme"
+	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
+	rest "k8s.io/client-go/rest"
+)
+
+type LotusV1beta1Interface interface {
+	RESTClient() rest.Interface
+	LotusesGetter
+}
+
+// LotusV1beta1Client is used to interact with features provided by the lotus.nghialv.com group.
+type LotusV1beta1Client struct {
+	restClient rest.Interface
+}
+
+func (c *LotusV1beta1Client) Lotuses(namespace string) LotusInterface {
+	return newLotuses(c, namespace)
+}
+
+// NewForConfig creates a new LotusV1beta1Client for the given config.
+func NewForConfig(c *rest.Config) (*LotusV1beta1Client, error) {
+	config := *c
+	if err := setConfigDefaults(&config); err != nil {
+		return nil, err
+	}
+	client, err := rest.RESTClientFor(&config)
+	if err != nil {
+		return nil, err
+	}
+	return &LotusV1beta1Client{client}, nil
+}
+
+// NewForConfigOrDie creates a new LotusV1beta1Client for the given config and
+// panics if there is an error in the config.
+func NewForConfigOrDie(c *rest.Config) *LotusV1beta1Client {
+	client, err := NewForConfig(c)
+	if err != nil {
+		panic(err)
+	}
+	return client
+}
+
+// New creates a new LotusV1beta1Client for the given RESTClient.
+func New(c rest.Interface) *LotusV1beta1Client {
+	return &LotusV1beta1Client{c}
+}
+
+func setConfigDefaults(config *rest.Config) error {
+	gv := v1beta1.SchemeGroupVersion
+	config.GroupVersion = &gv
+	config.APIPath = "/apis"
+	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+
+	if config.UserAgent == "" {
+		config.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+
+	return nil
+}
+
+// RESTClient returns a RESTClient that is used to communicate
+// with API server by this client implementation.
+func (c *LotusV1beta1Client) RESTClient() rest.Interface {
+	if c == nil {
+		return nil
+	}
+	return c.restClient
+}
diff --git a/pkg/app/lotus/client/informers/externalversions/BUILD.bazel b/pkg/app/lotus/client/informers/externalversions/BUILD.bazel
new file mode 100644
index 0000000..f2ce261
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/BUILD.bazel
@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "factory.go",
+        "generic.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions/internalinterfaces:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions/lotus:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_client_go//tools/cache:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/informers/externalversions/factory.go b/pkg/app/lotus/client/informers/externalversions/factory.go
new file mode 100644
index 0000000..cb9529d
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/factory.go
@@ -0,0 +1,170 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package externalversions
+
+import (
+	reflect "reflect"
+	sync "sync"
+	time "time"
+
+	versioned "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	internalinterfaces "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/internalinterfaces"
+	lotus "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/lotus"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// SharedInformerOption defines the functional option type for SharedInformerFactory.
+type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory
+
+type sharedInformerFactory struct {
+	client           versioned.Interface
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+	lock             sync.Mutex
+	defaultResync    time.Duration
+	customResync     map[reflect.Type]time.Duration
+
+	informers map[reflect.Type]cache.SharedIndexInformer
+	// startedInformers is used for tracking which informers have been started.
+	// This allows Start() to be called multiple times safely.
+	startedInformers map[reflect.Type]bool
+}
+
+// WithCustomResyncConfig sets a custom resync period for the specified informer types.
+func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		for k, v := range resyncConfig {
+			factory.customResync[reflect.TypeOf(k)] = v
+		}
+		return factory
+	}
+}
+
+// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory.
+func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		factory.tweakListOptions = tweakListOptions
+		return factory
+	}
+}
+
+// WithNamespace limits the SharedInformerFactory to the specified namespace.
+func WithNamespace(namespace string) SharedInformerOption {
+	return func(factory *sharedInformerFactory) *sharedInformerFactory {
+		factory.namespace = namespace
+		return factory
+	}
+}
+
+// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.
+func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory {
+	return NewSharedInformerFactoryWithOptions(client, defaultResync)
+}
+
+// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
+// Listers obtained via this SharedInformerFactory will be subject to the same filters
+// as specified here.
+// Deprecated: Please use NewSharedInformerFactoryWithOptions instead
+func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
+	return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions))
+}
+
+// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.
+func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
+	factory := &sharedInformerFactory{
+		client:           client,
+		namespace:        v1.NamespaceAll,
+		defaultResync:    defaultResync,
+		informers:        make(map[reflect.Type]cache.SharedIndexInformer),
+		startedInformers: make(map[reflect.Type]bool),
+		customResync:     make(map[reflect.Type]time.Duration),
+	}
+
+	// Apply all options
+	for _, opt := range options {
+		factory = opt(factory)
+	}
+
+	return factory
+}
+
+// Start initializes all requested informers.
+func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	for informerType, informer := range f.informers {
+		if !f.startedInformers[informerType] {
+			go informer.Run(stopCh)
+			f.startedInformers[informerType] = true
+		}
+	}
+}
+
+// WaitForCacheSync waits for all started informers' cache were synced.
+func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
+	informers := func() map[reflect.Type]cache.SharedIndexInformer {
+		f.lock.Lock()
+		defer f.lock.Unlock()
+
+		informers := map[reflect.Type]cache.SharedIndexInformer{}
+		for informerType, informer := range f.informers {
+			if f.startedInformers[informerType] {
+				informers[informerType] = informer
+			}
+		}
+		return informers
+	}()
+
+	res := map[reflect.Type]bool{}
+	for informType, informer := range informers {
+		res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced)
+	}
+	return res
+}
+
+// InternalInformerFor returns the SharedIndexInformer for obj using an internal
+// client.
+func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	informerType := reflect.TypeOf(obj)
+	informer, exists := f.informers[informerType]
+	if exists {
+		return informer
+	}
+
+	resyncPeriod, exists := f.customResync[informerType]
+	if !exists {
+		resyncPeriod = f.defaultResync
+	}
+
+	informer = newFunc(f.client, resyncPeriod)
+	f.informers[informerType] = informer
+
+	return informer
+}
+
+// SharedInformerFactory provides shared informers for resources in all known
+// API group versions.
+type SharedInformerFactory interface {
+	internalinterfaces.SharedInformerFactory
+	ForResource(resource schema.GroupVersionResource) (GenericInformer, error)
+	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
+
+	Lotus() lotus.Interface
+}
+
+func (f *sharedInformerFactory) Lotus() lotus.Interface {
+	return lotus.New(f, f.namespace, f.tweakListOptions)
+}
diff --git a/pkg/app/lotus/client/informers/externalversions/generic.go b/pkg/app/lotus/client/informers/externalversions/generic.go
new file mode 100644
index 0000000..025ddd5
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/generic.go
@@ -0,0 +1,52 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package externalversions
+
+import (
+	"fmt"
+
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// GenericInformer is type of SharedIndexInformer which will locate and delegate to other
+// sharedInformers based on type
+type GenericInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() cache.GenericLister
+}
+
+type genericInformer struct {
+	informer cache.SharedIndexInformer
+	resource schema.GroupResource
+}
+
+// Informer returns the SharedIndexInformer.
+func (f *genericInformer) Informer() cache.SharedIndexInformer {
+	return f.informer
+}
+
+// Lister returns the GenericLister.
+func (f *genericInformer) Lister() cache.GenericLister {
+	return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource)
+}
+
+// ForResource gives generic access to a shared informer of the matching type
+// TODO extend this to unknown resources with a client pool
+func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) {
+	switch resource {
+	// Group=lotus.nghialv.com, Version=v1beta1
+	case v1beta1.SchemeGroupVersion.WithResource("lotuses"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Lotus().V1beta1().Lotuses().Informer()}, nil
+
+	}
+
+	return nil, fmt.Errorf("no informer found for %v", resource)
+}
diff --git a/pkg/app/lotus/client/informers/externalversions/internalinterfaces/BUILD.bazel b/pkg/app/lotus/client/informers/externalversions/internalinterfaces/BUILD.bazel
new file mode 100644
index 0000000..95711eb
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/internalinterfaces/BUILD.bazel
@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["factory_interfaces.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/internalinterfaces",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_client_go//tools/cache:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/app/lotus/client/informers/externalversions/internalinterfaces/factory_interfaces.go
new file mode 100644
index 0000000..5d26dc1
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/internalinterfaces/factory_interfaces.go
@@ -0,0 +1,28 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package internalinterfaces
+
+import (
+	time "time"
+
+	versioned "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer
+
+// SharedInformerFactory a small interface to allow for adding an informer without an import cycle
+type SharedInformerFactory interface {
+	Start(stopCh <-chan struct{})
+	InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer
+}
+
+type TweakListOptionsFunc func(*v1.ListOptions)
diff --git a/pkg/app/lotus/client/informers/externalversions/lotus/BUILD.bazel b/pkg/app/lotus/client/informers/externalversions/lotus/BUILD.bazel
new file mode 100644
index 0000000..981951f
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/lotus/BUILD.bazel
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["interface.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/lotus",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/client/informers/externalversions/internalinterfaces:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions/lotus/v1beta1:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/informers/externalversions/lotus/interface.go b/pkg/app/lotus/client/informers/externalversions/lotus/interface.go
new file mode 100644
index 0000000..168e486
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/lotus/interface.go
@@ -0,0 +1,36 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package lotus
+
+import (
+	internalinterfaces "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/internalinterfaces"
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1"
+)
+
+// Interface provides access to each of this group's versions.
+type Interface interface {
+	// V1beta1 provides access to shared informers for resources in V1beta1.
+	V1beta1() v1beta1.Interface
+}
+
+type group struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// V1beta1 returns a new v1beta1.Interface.
+func (g *group) V1beta1() v1beta1.Interface {
+	return v1beta1.New(g.factory, g.namespace, g.tweakListOptions)
+}
diff --git a/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/BUILD.bazel b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/BUILD.bazel
new file mode 100644
index 0000000..584233f
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/BUILD.bazel
@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "interface.go",
+        "lotus.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions/internalinterfaces:go_default_library",
+        "//pkg/app/lotus/client/listers/lotus/v1beta1:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/watch:go_default_library",
+        "@io_k8s_client_go//tools/cache:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/interface.go b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/interface.go
new file mode 100644
index 0000000..781b29d
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/interface.go
@@ -0,0 +1,35 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	internalinterfaces "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/internalinterfaces"
+)
+
+// Interface provides access to all the informers in this group version.
+type Interface interface {
+	// Lotuses returns a LotusInformer.
+	Lotuses() LotusInformer
+}
+
+type version struct {
+	factory          internalinterfaces.SharedInformerFactory
+	namespace        string
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// New returns a new Interface.
+func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
+	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
+}
+
+// Lotuses returns a LotusInformer.
+func (v *version) Lotuses() LotusInformer {
+	return &lotusInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
+}
diff --git a/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/lotus.go b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/lotus.go
new file mode 100644
index 0000000..47af03b
--- /dev/null
+++ b/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1/lotus.go
@@ -0,0 +1,79 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	time "time"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	versioned "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	internalinterfaces "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/internalinterfaces"
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/client/listers/lotus/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// LotusInformer provides access to a shared informer and lister for
+// Lotuses.
+type LotusInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1beta1.LotusLister
+}
+
+type lotusInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+	namespace        string
+}
+
+// NewLotusInformer constructs a new informer for Lotus type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewLotusInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredLotusInformer(client, namespace, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredLotusInformer constructs a new informer for Lotus type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredLotusInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.LotusV1beta1().Lotuses(namespace).List(options)
+			},
+			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.LotusV1beta1().Lotuses(namespace).Watch(options)
+			},
+		},
+		&lotusv1beta1.Lotus{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *lotusInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredLotusInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *lotusInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&lotusv1beta1.Lotus{}, f.defaultInformer)
+}
+
+func (f *lotusInformer) Lister() v1beta1.LotusLister {
+	return v1beta1.NewLotusLister(f.Informer().GetIndexer())
+}
diff --git a/pkg/app/lotus/client/listers/lotus/v1beta1/BUILD.bazel b/pkg/app/lotus/client/listers/lotus/v1beta1/BUILD.bazel
new file mode 100644
index 0000000..b002c17
--- /dev/null
+++ b/pkg/app/lotus/client/listers/lotus/v1beta1/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "expansion_generated.go",
+        "lotus.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/client/listers/lotus/v1beta1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
+        "@io_k8s_apimachinery//pkg/labels:go_default_library",
+        "@io_k8s_client_go//tools/cache:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/client/listers/lotus/v1beta1/expansion_generated.go b/pkg/app/lotus/client/listers/lotus/v1beta1/expansion_generated.go
new file mode 100644
index 0000000..8936f89
--- /dev/null
+++ b/pkg/app/lotus/client/listers/lotus/v1beta1/expansion_generated.go
@@ -0,0 +1,17 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1beta1
+
+// LotusListerExpansion allows custom methods to be added to
+// LotusLister.
+type LotusListerExpansion interface{}
+
+// LotusNamespaceListerExpansion allows custom methods to be added to
+// LotusNamespaceLister.
+type LotusNamespaceListerExpansion interface{}
diff --git a/pkg/app/lotus/client/listers/lotus/v1beta1/lotus.go b/pkg/app/lotus/client/listers/lotus/v1beta1/lotus.go
new file mode 100644
index 0000000..79c8434
--- /dev/null
+++ b/pkg/app/lotus/client/listers/lotus/v1beta1/lotus.go
@@ -0,0 +1,84 @@
+/*
+
+Generated by using code-generator
+
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	v1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+)
+
+// LotusLister helps list Lotuses.
+type LotusLister interface {
+	// List lists all Lotuses in the indexer.
+	List(selector labels.Selector) (ret []*v1beta1.Lotus, err error)
+	// Lotuses returns an object that can list and get Lotuses.
+	Lotuses(namespace string) LotusNamespaceLister
+	LotusListerExpansion
+}
+
+// lotusLister implements the LotusLister interface.
+type lotusLister struct {
+	indexer cache.Indexer
+}
+
+// NewLotusLister returns a new LotusLister.
+func NewLotusLister(indexer cache.Indexer) LotusLister {
+	return &lotusLister{indexer: indexer}
+}
+
+// List lists all Lotuses in the indexer.
+func (s *lotusLister) List(selector labels.Selector) (ret []*v1beta1.Lotus, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.Lotus))
+	})
+	return ret, err
+}
+
+// Lotuses returns an object that can list and get Lotuses.
+func (s *lotusLister) Lotuses(namespace string) LotusNamespaceLister {
+	return lotusNamespaceLister{indexer: s.indexer, namespace: namespace}
+}
+
+// LotusNamespaceLister helps list and get Lotuses.
+type LotusNamespaceLister interface {
+	// List lists all Lotuses in the indexer for a given namespace.
+	List(selector labels.Selector) (ret []*v1beta1.Lotus, err error)
+	// Get retrieves the Lotus from the indexer for a given namespace and name.
+	Get(name string) (*v1beta1.Lotus, error)
+	LotusNamespaceListerExpansion
+}
+
+// lotusNamespaceLister implements the LotusNamespaceLister
+// interface.
+type lotusNamespaceLister struct {
+	indexer   cache.Indexer
+	namespace string
+}
+
+// List lists all Lotuses in the indexer for a given namespace.
+func (s lotusNamespaceLister) List(selector labels.Selector) (ret []*v1beta1.Lotus, err error) {
+	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.Lotus))
+	})
+	return ret, err
+}
+
+// Get retrieves the Lotus from the indexer for a given namespace and name.
+func (s lotusNamespaceLister) Get(name string) (*v1beta1.Lotus, error) {
+	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1beta1.Resource("lotus"), name)
+	}
+	return obj.(*v1beta1.Lotus), nil
+}
diff --git a/pkg/app/lotus/cmd/controller/BUILD.bazel b/pkg/app/lotus/cmd/controller/BUILD.bazel
new file mode 100644
index 0000000..4ee2b90
--- /dev/null
+++ b/pkg/app/lotus/cmd/controller/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["controller.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/cmd/controller",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions:go_default_library",
+        "//pkg/app/lotus/controller:go_default_library",
+        "//pkg/cli:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@io_k8s_client_go//informers:go_default_library",
+        "@io_k8s_client_go//kubernetes:go_default_library",
+        "@io_k8s_client_go//plugin/pkg/client/auth/gcp:go_default_library",
+        "@io_k8s_client_go//tools/clientcmd:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/cmd/controller/controller.go b/pkg/app/lotus/cmd/controller/controller.go
new file mode 100644
index 0000000..6b35115
--- /dev/null
+++ b/pkg/app/lotus/cmd/controller/controller.go
@@ -0,0 +1,102 @@
+package controller
+
+import (
+	"context"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/tools/clientcmd"
+
+	clientset "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	informers "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions"
+	lotus "github.com/nghialv/lotus/pkg/app/lotus/controller"
+	"github.com/nghialv/lotus/pkg/cli"
+)
+
+type controller struct {
+	kubeconfig               string
+	masterURL                string
+	namespace                string
+	release                  string
+	prometheusServiceAccount string
+	configFile               string
+}
+
+func NewCommand() *cobra.Command {
+	c := &controller{
+		namespace: "default",
+		release:   "lotus",
+	}
+	cmd := &cobra.Command{
+		Use:   "controller",
+		Short: "Start running Lotus controller",
+		RunE:  cli.WithContext(c.run),
+	}
+	cmd.Flags().StringVar(&c.kubeconfig, "kube-config", c.kubeconfig, "Path to a kubeconfig. Only required if out-of-cluster.")
+	cmd.Flags().StringVar(&c.masterURL, "master", c.masterURL, "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+	cmd.Flags().StringVar(&c.namespace, "namespace", c.namespace, "The namespace of controller.")
+	cmd.Flags().StringVar(&c.release, "release", c.release, "The release name of deployment.")
+	cmd.Flags().StringVar(&c.prometheusServiceAccount, "prometheus-service-account", c.prometheusServiceAccount, "The name of service account for prometheus pods. This is required when rbac is enabled.")
+	cmd.Flags().StringVar(&c.configFile, "config-file", c.configFile, "Path to the configuration file.")
+	cmd.MarkFlagRequired("config-file")
+	return cmd
+}
+
+func (c *controller) run(ctx context.Context, logger *zap.Logger) error {
+	cfg, err := clientcmd.BuildConfigFromFlags(c.masterURL, c.kubeconfig)
+	if err != nil {
+		logger.Error("failed to build kube config", zap.Error(err))
+		return err
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		logger.Error("failed to build kubernetes clientset", zap.Error(err))
+		return err
+	}
+
+	lotusClient, err := clientset.NewForConfig(cfg)
+	if err != nil {
+		logger.Error("failed to build lotus clientset", zap.Error(err))
+		return err
+	}
+
+	kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
+		kubeClient,
+		30*time.Second,
+		kubeinformers.WithNamespace(c.namespace),
+	)
+
+	lotusInformerFactory := informers.NewSharedInformerFactoryWithOptions(
+		lotusClient,
+		30*time.Second,
+		informers.WithNamespace(c.namespace),
+	)
+
+	controller := lotus.NewController(
+		kubeClient,
+		lotusClient,
+		kubeInformerFactory.Batch().V1().Jobs(),
+		lotusInformerFactory.Lotus().V1beta1().Lotuses(),
+		c.namespace,
+		c.release,
+		c.prometheusServiceAccount,
+		c.configFile,
+		logger,
+	)
+
+	kubeInformerFactory.Start(ctx.Done())
+	lotusInformerFactory.Start(ctx.Done())
+
+	if err = controller.Run(ctx, 1); err != nil {
+		logger.Error("failed to run controller", zap.Error(err))
+		return err
+	}
+
+	<-ctx.Done()
+	return nil
+}
diff --git a/pkg/app/lotus/cmd/monitor/BUILD.bazel b/pkg/app/lotus/cmd/monitor/BUILD.bazel
new file mode 100644
index 0000000..2b57b56
--- /dev/null
+++ b/pkg/app/lotus/cmd/monitor/BUILD.bazel
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["monitor.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/cmd/monitor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/datasource:go_default_library",
+        "//pkg/app/lotus/datasource/registry:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/app/lotus/reporter:go_default_library",
+        "//pkg/app/lotus/reporter/registry:go_default_library",
+        "//pkg/cli:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/cmd/monitor/monitor.go b/pkg/app/lotus/cmd/monitor/monitor.go
new file mode 100644
index 0000000..b983a7c
--- /dev/null
+++ b/pkg/app/lotus/cmd/monitor/monitor.go
@@ -0,0 +1,239 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/datasource"
+	dsregistry "github.com/nghialv/lotus/pkg/app/lotus/datasource/registry"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter"
+	reporterregistry "github.com/nghialv/lotus/pkg/app/lotus/reporter/registry"
+	"github.com/nghialv/lotus/pkg/cli"
+)
+
+type monitor struct {
+	testID                   string
+	runTime                  time.Duration
+	checkInterval            time.Duration
+	checkInitialDelay        time.Duration
+	collectSummaryDataSource string
+	collectAndReportTimeout  time.Duration
+	configFile               string
+
+	dataSourceMap map[string]datasource.DataSource
+	checkMap      map[string][]datasource.Check
+	cfg           *config.Config
+	logger        *zap.Logger
+}
+
+func NewCommand() *cobra.Command {
+	m := &monitor{
+		runTime:                 2 * time.Minute,
+		checkInterval:           30 * time.Second,
+		checkInitialDelay:       10 * time.Second,
+		collectAndReportTimeout: 30 * time.Minute,
+	}
+	cmd := &cobra.Command{
+		Use:   "monitor",
+		Short: "Start running Lotus monitor",
+		RunE:  cli.WithContext(m.run),
+	}
+	cmd.Flags().StringVar(&m.testID, "test-id", m.testID, "The unique test id")
+	cmd.MarkFlagRequired("test-id")
+	cmd.Flags().DurationVar(&m.runTime, "run-time", m.runTime, "How long the worker should be run")
+	cmd.Flags().DurationVar(&m.checkInterval, "check-interval", m.checkInterval, "How often does the monitor run the check")
+	cmd.Flags().DurationVar(&m.checkInitialDelay, "check-initial-delay", m.checkInitialDelay, "How long the monitor should wait before performing the first check")
+	cmd.Flags().StringVar(&m.collectSummaryDataSource, "collect-summary-datasource", m.collectSummaryDataSource, "The datasource used to collect test summary")
+	cmd.MarkFlagRequired("collect-summary-datasource")
+	cmd.Flags().DurationVar(&m.collectAndReportTimeout, "collect-and-report-timeout", m.collectAndReportTimeout, "How log to wait for collect and report tasks")
+	cmd.Flags().StringVar(&m.configFile, "config-file", m.configFile, "Path to the configuration file")
+	cmd.MarkFlagRequired("config-file")
+	return cmd
+}
+
+func (m *monitor) run(ctx context.Context, logger *zap.Logger) (lastErr error) {
+	startTime := time.Now()
+	m.logger = logger.Named("monitor")
+	ctx, cancel := context.WithTimeout(ctx, m.runTime)
+	defer cancel()
+
+	defer func() {
+		if err := m.collectAndReport(startTime, time.Now(), lastErr); err != nil {
+			lastErr = err
+		}
+	}()
+
+	cfg, err := config.FromFile(m.configFile)
+	if err != nil {
+		logger.Error("failed to load configuration", zap.Error(err))
+		lastErr = err
+		return
+	}
+	m.cfg = cfg
+	dataSourceMap, err := buildDataSourceMap(cfg, logger)
+	if err != nil {
+		logger.Error("failed to build dataSourceMap", zap.Error(err))
+		lastErr = err
+		return
+	}
+	m.dataSourceMap = dataSourceMap
+	m.checkMap = buildCheckMap(cfg)
+
+	// Waiting for initial delay
+	select {
+	case <-time.After(m.checkInitialDelay):
+	case <-ctx.Done():
+	}
+
+	tick := time.Tick(m.checkInterval)
+	for {
+		select {
+		case <-tick:
+			lastErr = m.check(ctx)
+			if lastErr != nil {
+				return
+			}
+		case <-ctx.Done():
+			m.logger.Info("breaking the check loop due to the context deadline")
+			return
+		}
+	}
+}
+
+func (m *monitor) check(ctx context.Context) error {
+	actives := make([]string, 0)
+	m.logger.Info("start checking all datasources", zap.Int("num", len(m.dataSourceMap)))
+	for dsn, checks := range m.checkMap {
+		ds, ok := m.dataSourceMap[dsn]
+		if !ok {
+			err := fmt.Errorf("missing datasource: %s", dsn)
+			m.logger.Error("failed to get datasource", zap.Error(err))
+			return err
+		}
+		result, err := ds.Check(ctx, checks)
+		if err != nil {
+			m.logger.Error("failed to check", zap.Error(err))
+			return err
+		}
+		actives = append(actives, result.Actives...)
+	}
+	if len(actives) == 0 {
+		return nil
+	}
+	m.logger.Info("active checks", zap.Any("actives", actives))
+	return checkError{
+		Actives: actives,
+	}
+}
+
+type checkError struct {
+	Actives []string
+}
+
+func (ce checkError) Error() string {
+	return fmt.Sprintf("%d checks are failed", len(ce.Actives))
+}
+
+func (m *monitor) collectAndReport(startTime, finishTime time.Time, lastErr error) error {
+	ctx, cancel := context.WithTimeout(context.Background(), m.collectAndReportTimeout)
+	defer cancel()
+	result := &model.Result{
+		TestID:            m.testID,
+		Status:            model.TestSucceeded,
+		StartedTimestamp:  startTime,
+		FinishedTimestamp: finishTime,
+	}
+	if lastErr != nil {
+		result.SetFailed(lastErr.Error())
+	}
+	if ce, ok := lastErr.(checkError); ok {
+		result.FailedChecks = ce.Actives
+	}
+
+	summary, collectErr := m.collect(ctx)
+	if collectErr != nil {
+		m.logger.Error("failed to collect metrics summary", zap.Error(collectErr))
+		if result.Status != model.TestFailed {
+			result.SetFailed("failed to collect metrics summary")
+		}
+	} else {
+		result.MetricsSummary = summary
+	}
+	if m.cfg != nil {
+		result.SetGrafanaDashboardURLs(m.cfg.GrafanaBaseUrl)
+	}
+	if err := m.report(ctx, result); err != nil {
+		m.logger.Error("failed to report result", zap.Error(err))
+		return err
+	}
+	return collectErr
+}
+
+func (m *monitor) collect(ctx context.Context) (*model.MetricsSummary, error) {
+	ds, ok := m.dataSourceMap[m.collectSummaryDataSource]
+	if !ok {
+		err := fmt.Errorf("missing datasource for collecting test summary: %s", m.collectSummaryDataSource)
+		m.logger.Error("failed to get datasource", zap.Error(err))
+		return nil, err
+	}
+	return ds.CollectSummary(ctx, time.Now())
+}
+
+func (m *monitor) report(ctx context.Context, result *model.Result) error {
+	rs := make([]reporter.Reporter, 0, len(m.cfg.Receivers))
+	for _, recv := range m.cfg.Receivers {
+		builder, err := reporterregistry.Default().Get(recv.ReceiverType())
+		if err != nil {
+			return err
+		}
+		r, err := builder.Build(recv, reporter.BuildOptions{
+			Logger: m.logger,
+		})
+		if err != nil {
+			return err
+		}
+		rs = append(rs, r)
+	}
+	return reporter.MultiReporter(rs...).Report(ctx, result)
+}
+
+func buildDataSourceMap(cfg *config.Config, logger *zap.Logger) (map[string]datasource.DataSource, error) {
+	datasources := make(map[string]datasource.DataSource, len(cfg.DataSources))
+	for _, ds := range cfg.DataSources {
+		builder, err := dsregistry.Default().Get(ds.DataSourceType())
+		if err != nil {
+			return nil, err
+		}
+		datasource, err := builder.Build(ds, datasource.BuildOptions{
+			Logger: logger,
+		})
+		if err != nil {
+			return nil, err
+		}
+		datasources[ds.Name] = datasource
+	}
+	return datasources, nil
+}
+
+func buildCheckMap(cfg *config.Config) map[string][]datasource.Check {
+	checkMap := make(map[string][]datasource.Check)
+	for _, check := range cfg.Checks {
+		c := datasource.Check{
+			Name: check.Name,
+			Expr: check.Expr,
+			For:  check.For,
+		}
+		if list, ok := checkMap[check.DataSource]; ok {
+			checkMap[check.DataSource] = append(list, c)
+			continue
+		}
+		checkMap[check.DataSource] = []datasource.Check{c}
+	}
+	return checkMap
+}
diff --git a/pkg/app/lotus/config/BUILD.bazel b/pkg/app/lotus/config/BUILD.bazel
new file mode 100644
index 0000000..da07069
--- /dev/null
+++ b/pkg/app/lotus/config/BUILD.bazel
@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("//:pgv_proto_library.bzl", "pgv_go_proto_library")
+
+pgv_go_proto_library(
+    name = "config_go_proto",
+    proto = ":config_proto",
+    importpath = "github.com/nghialv/lotus/pkg/config",
+    deps = [
+        "@com_github_golang_protobuf//ptypes:go_default_library_gen",
+    ],
+)
+
+proto_library(
+    name = "config_proto",
+    srcs = ["config.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["@com_lyft_protoc_gen_validate//validate:validate_proto"],  #keep
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    embed = [":config_go_proto"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/config",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+        "@com_github_golang_protobuf//jsonpb:go_default_library_gen",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["config_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = [
+        "@com_github_stretchr_testify//assert:go_default_library",
+        "@com_github_stretchr_testify//require:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/config/config.go b/pkg/app/lotus/config/config.go
new file mode 100644
index 0000000..65bdc66
--- /dev/null
+++ b/pkg/app/lotus/config/config.go
@@ -0,0 +1,97 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+	"github.com/golang/protobuf/jsonpb"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+)
+
+func (c *Config) AddChecks(checks ...lotusv1beta1.LotusCheck) {
+	for i := range checks {
+		c.Checks = append(c.Checks, &Check{
+			Name:       checks[i].Name,
+			Expr:       checks[i].Expr,
+			For:        checks[i].For,
+			DataSource: checks[i].DataSource,
+		})
+	}
+}
+
+func (c *Config) LotusChecks() []lotusv1beta1.LotusCheck {
+	checks := make([]lotusv1beta1.LotusCheck, 0, len(c.Checks))
+	for _, check := range c.Checks {
+		checks = append(checks, lotusv1beta1.LotusCheck{
+			Name:       check.Name,
+			Expr:       check.Expr,
+			For:        check.For,
+			DataSource: check.DataSource,
+		})
+	}
+	return checks
+}
+
+func (ds *DataSource) DataSourceType() DataSource_Type {
+	switch ds.Type.(type) {
+	case *DataSource_Prometheus:
+		return DataSource_PROMETHEUS
+	default:
+		return DataSource_UNKNOWN
+	}
+}
+
+func (r *Receiver) ReceiverType() Receiver_Type {
+	switch r.Type.(type) {
+	case *Receiver_Logger:
+		return Receiver_LOGGER
+	case *Receiver_Gcs:
+		return Receiver_GCS
+	case *Receiver_Slack:
+		return Receiver_SLACK
+	default:
+		return Receiver_UNKNOWN
+	}
+}
+
+func (r *Receiver) CredentialsMountPath() string {
+	return fmt.Sprintf("/etc/creds/%s/", r.Name)
+}
+
+func (r *Receiver) CredentialsFile(filename string) string {
+	return fmt.Sprintf("%s%s", r.CredentialsMountPath(), filename)
+}
+
+func FromFile(file string) (*Config, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	return UnmarshalFromYaml(data)
+}
+
+func UnmarshalFromYaml(data []byte) (*Config, error) {
+	json, err := yaml.YAMLToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	config := &Config{}
+	if err = jsonpb.UnmarshalString(string(json), config); err != nil {
+		return nil, err
+	}
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+func (c *Config) MarshalToYaml() ([]byte, error) {
+	marshaler := &jsonpb.Marshaler{}
+	json, err := marshaler.MarshalToString(c)
+	if err != nil {
+		return nil, err
+	}
+	return yaml.JSONToYAML([]byte(json))
+}
diff --git a/pkg/app/lotus/config/config.proto b/pkg/app/lotus/config/config.proto
new file mode 100644
index 0000000..c58f254
--- /dev/null
+++ b/pkg/app/lotus/config/config.proto
@@ -0,0 +1,99 @@
+syntax = "proto3";
+
+package pkg.lotus.config;
+option go_package = "config";
+
+import "validate/validate.proto";
+
+message Config {
+  repeated DataSource data_sources = 1;
+  repeated Check checks = 2;
+  repeated Receiver receivers = 3;
+  TimeSeriesStorage time_series_storage = 4;
+  string grafana_base_url = 5;
+}
+
+message TimeSeriesStorage {
+  enum Type {
+    GCS = 0;
+    AWS_S3 = 1;
+  }
+  oneof type {
+    option (validate.required) = true;
+    GCSTimeSeriesStorageConfigs gcs = 10;
+    S3TimeSeriesStorageConfigs s3 = 11;
+  }
+}
+
+message GCSTimeSeriesStorageConfigs {
+  string bucket = 1 [(validate.rules).string.min_len = 1];
+  SecretFileSelector credentials = 2;
+}
+
+message S3TimeSeriesStorageConfigs {
+  string bucket = 1 [(validate.rules).string.min_len = 1];
+  string endpoint = 2 [(validate.rules).string.min_len = 1];
+  SecretFileSelector access_key = 3 [(validate.rules).message.required = true];
+  SecretFileSelector secret_key = 4 [(validate.rules).message.required = true];
+  bool insecure = 5;
+  bool signature_version2 = 6;
+  bool encrypt_sse = 7;
+}
+
+message Check {
+  string name = 1 [(validate.rules).string.min_len = 1];
+  string expr = 2 [(validate.rules).string.min_len = 1];
+  string for = 3 [(validate.rules).string.min_len = 1];
+  string data_source = 4;
+}
+
+message DataSource {
+  enum Type {
+    PROMETHEUS = 0;
+    STACKDRIVER = 1;
+    DATADOG = 2;
+    UNKNOWN = 15;
+  }
+  string name = 1 [(validate.rules).string.min_len = 1];
+  oneof type {
+    option (validate.required) = true;
+    PrometheusConfigs prometheus = 10;
+  }
+}
+
+message PrometheusConfigs {
+  string address = 1 [(validate.rules).string.uri = true];
+}
+
+message Receiver {
+  enum Type {
+    LOGGER = 0;
+    GCS = 1;
+    SLACK = 2;
+    UNKNOWN = 15;
+  }
+  string name = 1 [(validate.rules).string.min_len = 1];
+  oneof type {
+    option (validate.required) = true;
+    LoggerReceiverConfigs logger = 10;
+    GCSReceiverConfigs gcs = 11;
+    SlackReceiverConfigs slack = 12;
+  }
+}
+
+message LoggerReceiverConfigs {
+}
+
+message GCSReceiverConfigs {
+  string bucket = 1 [(validate.rules).string.min_len = 1];
+  SecretFileSelector credentials = 2;
+}
+
+message SlackReceiverConfigs {
+  string hook_url = 1 [(validate.rules).string.uri = true];
+}
+
+message SecretFileSelector {
+  string secret = 1 [(validate.rules).string.min_len = 1];
+  string file = 2 [(validate.rules).string.min_len =1];
+}
diff --git a/pkg/app/lotus/config/config_test.go b/pkg/app/lotus/config/config_test.go
new file mode 100644
index 0000000..6045505
--- /dev/null
+++ b/pkg/app/lotus/config/config_test.go
@@ -0,0 +1,97 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFromFile(t *testing.T) {
+	cfg, err := FromFile("testdata/valid.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	require.NotNil(t, cfg.TimeSeriesStorage)
+	gcs, ok := cfg.TimeSeriesStorage.Type.(*TimeSeriesStorage_Gcs)
+	require.True(t, ok)
+	assert.Equal(t, "gcs-bucket", gcs.Gcs.Bucket)
+	assert.NotNil(t, gcs.Gcs.Credentials)
+	assert.Equal(t, 1, len(cfg.DataSources))
+	assert.Equal(t, 1, len(cfg.Checks))
+	assert.Equal(t, 3, len(cfg.Receivers))
+}
+
+func TestMarshaling(t *testing.T) {
+	configs := []*Config{
+		&Config{},
+		&Config{
+			DataSources: []*DataSource{
+				&DataSource{
+					Name: "prometheus",
+					Type: &DataSource_Prometheus{
+						Prometheus: &PrometheusConfigs{
+							Address: "https://127.0.0.1:9090",
+						},
+					},
+				},
+			},
+			Checks: []*Check{
+				&Check{
+					Name: "HighErrorRate",
+					Expr: "error_rate > 0.5",
+					For:  "1m",
+				},
+				&Check{
+					Name:       "HighLatency",
+					Expr:       "latency > 125",
+					For:        "1m",
+					DataSource: "prometheus",
+				},
+			},
+			Receivers: []*Receiver{
+				&Receiver{
+					Name: "gcs",
+					Type: &Receiver_Gcs{
+						Gcs: &GCSReceiverConfigs{
+							Bucket: "bucket-2",
+							Credentials: &SecretFileSelector{
+								Secret: "foo",
+								File:   "credentials-2",
+							},
+						},
+					},
+				},
+				&Receiver{
+					Name: "slack",
+					Type: &Receiver_Slack{
+						Slack: &SlackReceiverConfigs{
+							HookUrl: "http://api-2.slack.com",
+						},
+					},
+				},
+			},
+			TimeSeriesStorage: &TimeSeriesStorage{
+				Type: &TimeSeriesStorage_Gcs{
+					Gcs: &GCSTimeSeriesStorageConfigs{
+						Bucket: "gcs-bucket",
+						Credentials: &SecretFileSelector{
+							Secret: "secret-name",
+							File:   "filename",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, cfg := range configs {
+		require.NoError(t, cfg.Validate())
+
+		data, err := cfg.MarshalToYaml()
+		require.NoError(t, err)
+
+		unmarshaledCfg, err := UnmarshalFromYaml(data)
+		require.NoError(t, err)
+		assert.Equal(t, cfg, unmarshaledCfg)
+	}
+}
diff --git a/pkg/app/lotus/config/testdata/valid.yaml b/pkg/app/lotus/config/testdata/valid.yaml
new file mode 100644
index 0000000..8b73686
--- /dev/null
+++ b/pkg/app/lotus/config/testdata/valid.yaml
@@ -0,0 +1,26 @@
+timeSeriesStorage:
+  gcs:
+    bucket: gcs-bucket
+    credentials:
+      secret: gcs-credentials
+      file: gcs-credentials.json
+dataSources:
+  - name: RemotePrometheus
+    prometheus:
+      address: http://prometheus.com
+checks:
+  - name: NoWorker
+    expr: absent(up)
+    for: 30s
+receivers:
+  - name: gcs
+    gcs:
+      bucket: load-testing-result
+      credentials:
+        secret: secret-name
+        file: filename
+  - name: slack
+    slack:
+      hookUrl: https://slack.com/hook
+  - name: logger
+    logger:
diff --git a/pkg/app/lotus/controller/BUILD.bazel b/pkg/app/lotus/controller/BUILD.bazel
new file mode 100644
index 0000000..5370b83
--- /dev/null
+++ b/pkg/app/lotus/controller/BUILD.bazel
@@ -0,0 +1,41 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["controller.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/controller",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned:go_default_library",
+        "//pkg/app/lotus/client/clientset/versioned/scheme:go_default_library",
+        "//pkg/app/lotus/client/informers/externalversions/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/client/listers/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/kubeclient:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/app/lotus/resource:go_default_library",
+        "@io_k8s_api//apps/v1:go_default_library",
+        "@io_k8s_api//core/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/wait:go_default_library",
+        "@io_k8s_client_go//informers/batch/v1:go_default_library",
+        "@io_k8s_client_go//kubernetes:go_default_library",
+        "@io_k8s_client_go//kubernetes/scheme:go_default_library",
+        "@io_k8s_client_go//kubernetes/typed/core/v1:go_default_library",
+        "@io_k8s_client_go//tools/cache:go_default_library",
+        "@io_k8s_client_go//tools/record:go_default_library",
+        "@io_k8s_client_go//util/workqueue:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["controller_test.go"],
+    embed = [":go_default_library"],
+)
diff --git a/pkg/app/lotus/controller/controller.go b/pkg/app/lotus/controller/controller.go
new file mode 100644
index 0000000..1331c9f
--- /dev/null
+++ b/pkg/app/lotus/controller/controller.go
@@ -0,0 +1,466 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	batchinformers "k8s.io/client-go/informers/batch/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/workqueue"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	clientset "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned"
+	lotusscheme "github.com/nghialv/lotus/pkg/app/lotus/client/clientset/versioned/scheme"
+	informers "github.com/nghialv/lotus/pkg/app/lotus/client/informers/externalversions/lotus/v1beta1"
+	listers "github.com/nghialv/lotus/pkg/app/lotus/client/listers/lotus/v1beta1"
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/kubeclient"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/app/lotus/resource"
+)
+
+type Controller struct {
+	kubeClient     kubeclient.KubeClient
+	lotusclientset clientset.Interface
+
+	jobsSynced    cache.InformerSynced
+	lotusesLister listers.LotusLister
+	lotusesSynced cache.InformerSynced
+
+	workqueue workqueue.RateLimitingInterface
+	recorder  record.EventRecorder
+
+	namespace                string
+	release                  string
+	prometheusServiceAccount string
+	configFile               string
+	logger                   *zap.Logger
+}
+
+func NewController(
+	kubeclientset kubernetes.Interface,
+	lotusclientset clientset.Interface,
+	jobInformer batchinformers.JobInformer,
+	lotusInformer informers.LotusInformer,
+	namespace string,
+	release string,
+	prometheusServiceAccount string,
+	configFile string,
+	logger *zap.Logger) *Controller {
+
+	logger = logger.Named("controller")
+	logger.Info("creating event broadcaster")
+	lotusscheme.AddToScheme(scheme.Scheme)
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(logger.Sugar().Infof)
+	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{
+		Interface: kubeclientset.CoreV1().Events(""),
+	})
+	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{
+		Component: "lotus-controller",
+	})
+
+	controller := &Controller{
+		kubeClient:               kubeclient.New(kubeclientset, jobInformer.Lister()),
+		lotusclientset:           lotusclientset,
+		jobsSynced:               jobInformer.Informer().HasSynced,
+		lotusesLister:            lotusInformer.Lister(),
+		lotusesSynced:            lotusInformer.Informer().HasSynced,
+		workqueue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Lotuses"),
+		recorder:                 recorder,
+		namespace:                namespace,
+		release:                  release,
+		prometheusServiceAccount: prometheusServiceAccount,
+		configFile:               configFile,
+		logger:                   logger,
+	}
+	lotusInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: controller.enqueueLotus,
+		UpdateFunc: func(old, new interface{}) {
+			controller.enqueueLotus(new)
+		},
+	})
+	jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(old, new interface{}) {
+			controller.onObject(new)
+		},
+	})
+	return controller
+}
+
+func (c *Controller) Run(ctx context.Context, workers int) error {
+	defer runtime.HandleCrash()
+	defer c.workqueue.ShutDown()
+
+	c.logger.Info("starting Lotus controller")
+	c.logger.Info("waiting for informer caches to sync")
+	if ok := cache.WaitForCacheSync(ctx.Done(), c.jobsSynced, c.lotusesSynced); !ok {
+		return fmt.Errorf("failed to wait for caches to sync")
+	}
+
+	// Update static resources based on new configuration
+	c.logger.Info("updating static resources")
+	if err := c.ensureStaticResources(); err != nil {
+		c.logger.Error("failed to ensure static resources", zap.Error(err))
+		return err
+	}
+
+	c.logger.Info("informer caches synced")
+	c.logger.Info("starting workers")
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.runWorker, time.Second, ctx.Done())
+	}
+
+	c.logger.Info("started workers", zap.Int("workers", workers))
+	<-ctx.Done()
+	c.logger.Info("shutting down workers")
+	return nil
+}
+
+func (c *Controller) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *Controller) processNextWorkItem() bool {
+	obj, shutdown := c.workqueue.Get()
+	if shutdown {
+		return false
+	}
+	err := func(obj interface{}) error {
+		defer c.workqueue.Done(obj)
+		key, ok := obj.(string)
+		if !ok {
+			c.workqueue.Forget(obj)
+			runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
+			return nil
+		}
+		if err := c.syncHandler(key); err != nil {
+			// Put the item back on the workqueue to handle any transient errors.
+			c.workqueue.AddRateLimited(key)
+			return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
+		}
+		// Finally, if no error occurs we Forget this item so it does not
+		// get queued again until another change happens.
+		c.workqueue.Forget(obj)
+		c.logger.Info("successfully synced item", zap.String("key", key))
+		return nil
+	}(obj)
+
+	if err != nil {
+		runtime.HandleError(err)
+		return true
+	}
+	return true
+}
+
+// Compares the actual state with the desired and attempts to converge the two.
+// It then updates the Status block of the Lotus resource with the current status of the resource.
+func (c *Controller) syncHandler(key string) error {
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
+		return nil
+	}
+	lotus, err := c.lotusesLister.Lotuses(namespace).Get(name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			runtime.HandleError(fmt.Errorf("lotus '%s' in work queue no longer exists", key))
+			return nil
+		}
+		return err
+	}
+
+	switch lotus.Status.Phase {
+	case lotusv1beta1.LotusInit:
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusPending)
+	case lotusv1beta1.LotusPending:
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusPreparing)
+	case lotusv1beta1.LotusPreparing:
+		if lotus.Spec.Preparer == nil {
+			return c.toRunningPhase(lotus)
+		}
+		return c.syncPreparingLotus(lotus)
+	case lotusv1beta1.LotusRunning:
+		return c.syncRunningLotus(lotus)
+	case lotusv1beta1.LotusCleaning:
+		if lotus.Spec.Cleaner == nil {
+			return c.updateLotusStatus(lotus, lotusv1beta1.LotusSucceeded)
+		}
+		return c.syncCleaningLotus(lotus)
+	case lotusv1beta1.LotusFailureCleaning:
+		if lotus.Spec.Cleaner == nil {
+			return c.updateLotusStatus(lotus, lotusv1beta1.LotusFailed)
+		}
+		return c.syncFailureCleaningLotus(lotus)
+	case lotusv1beta1.LotusSucceeded:
+		return nil
+	case lotusv1beta1.LotusFailed:
+		return nil
+	}
+	c.logger.Warn("unexpected lotus phase", zap.String("phase", string(lotus.Status.Phase)))
+	return nil
+}
+
+func (c *Controller) syncPreparingLotus(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	jobName := factory.PreparerJobName()
+	job, err := c.kubeClient.EnsureJob(jobName, lotus.Namespace, factory.NewPreparerJob)
+	if err != nil {
+		return err
+	}
+	if job.Status.Failed > 0 {
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusFailureCleaning)
+	}
+	if job.Status.Succeeded > 0 {
+		return c.toRunningPhase(lotus)
+	}
+	c.logger.Info("preparer job is still running", zap.String("name", jobName))
+	return nil
+}
+
+func (c *Controller) toRunningPhase(lotus *lotusv1beta1.Lotus) error {
+	if err := c.ensurePrometheusResources(lotus); err != nil {
+		return err
+	}
+	if err := c.ensureWorkerResources(lotus); err != nil {
+		return err
+	}
+	factory := resource.NewFactory(lotus, c.configFile)
+	name := factory.MonitorJobName()
+	if _, err := c.kubeClient.EnsureConfigMap(name, lotus.Namespace, factory.NewMonitorConfigMap); err != nil {
+		return err
+	}
+	return c.updateLotusStatus(lotus, lotusv1beta1.LotusRunning)
+}
+
+func (c *Controller) syncRunningLotus(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	jobName := factory.MonitorJobName()
+	job, err := c.kubeClient.EnsureJob(jobName, lotus.Namespace, factory.NewMonitorJob)
+	if err != nil {
+		return err
+	}
+	if job.Status.Succeeded == 0 && job.Status.Failed == 0 {
+		c.logger.Info("monitor job is still running", zap.String("name", jobName))
+		return nil
+	}
+	// Scale down or Delete worker deployment.
+	workerName := factory.WorkerName()
+	err = c.kubeClient.DeleteDeployment(workerName, lotus.Namespace)
+	if err != nil {
+		c.logger.Error("failed to delete worker deployment", zap.Error(err))
+		return err
+	}
+	if job.Status.Failed > 0 {
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusFailureCleaning)
+	}
+	return c.updateLotusStatus(lotus, lotusv1beta1.LotusCleaning)
+}
+
+func (c *Controller) syncCleaningLotus(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	jobName := factory.CleanerJobName()
+	job, err := c.kubeClient.EnsureJob(jobName, lotus.Namespace, factory.NewCleanerJob)
+	if err != nil {
+		return err
+	}
+	if job.Status.Succeeded > 0 {
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusSucceeded)
+	}
+	if job.Status.Failed > 0 {
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusFailed)
+	}
+	return nil
+}
+
+func (c *Controller) syncFailureCleaningLotus(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	jobName := factory.CleanerJobName()
+	job, err := c.kubeClient.EnsureJob(jobName, lotus.Namespace, factory.NewCleanerJob)
+	if err != nil {
+		return err
+	}
+	if job.Status.Succeeded > 0 || job.Status.Failed > 0 {
+		return c.updateLotusStatus(lotus, lotusv1beta1.LotusFailed)
+	}
+	return nil
+}
+
+func (c *Controller) ensureWorkerResources(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	name := factory.WorkerName()
+	if _, err := c.kubeClient.EnsureService(name, lotus.Namespace, factory.NewWorkerService); err != nil {
+		return err
+	}
+	_, err := c.kubeClient.EnsureDeployment(name, lotus.Namespace, factory.NewWorkerDeployment)
+	return err
+}
+
+func (c *Controller) ensurePrometheusResources(lotus *lotusv1beta1.Lotus) error {
+	factory := resource.NewFactory(lotus, c.configFile)
+	name := factory.PrometheusName()
+	if _, err := c.kubeClient.EnsureConfigMap(name, lotus.Namespace, factory.NewPrometheusConfigMap); err != nil {
+		return err
+	}
+	podFactory := func() (*corev1.Pod, error) {
+		return factory.NewPrometheusPod(c.prometheusServiceAccount, c.release)
+	}
+	if _, err := c.kubeClient.EnsurePod(name, lotus.Namespace, podFactory); err != nil {
+		return err
+	}
+	_, err := c.kubeClient.EnsureService(name, lotus.Namespace, factory.NewPrometheusService)
+	return err
+}
+
+func (c *Controller) enqueueLotus(obj interface{}) {
+	key, err := cache.MetaNamespaceKeyFunc(obj)
+	if err != nil {
+		runtime.HandleError(err)
+		return
+	}
+	c.logger.Info("enqueue a lotus", zap.String("key", key))
+	c.workqueue.AddRateLimited(key)
+}
+
+func (c *Controller) onObject(obj interface{}) {
+	object, ok := obj.(metav1.Object)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			runtime.HandleError(fmt.Errorf("error decoding object, invalid type"))
+			return
+		}
+		object, ok := tombstone.Obj.(metav1.Object)
+		if !ok {
+			runtime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type"))
+			return
+		}
+		c.logger.Info("recovered deleted object from tombstone", zap.String("name", object.GetName()))
+	}
+	ownerRef := metav1.GetControllerOf(object)
+	if ownerRef == nil {
+		return
+	}
+	if ownerRef.Kind != model.LotusKind {
+		return
+	}
+	lotus, err := c.lotusesLister.Lotuses(object.GetNamespace()).Get(ownerRef.Name)
+	if err != nil {
+		c.logger.Info("ignoring orphaned object",
+			zap.String("object_self_link", object.GetSelfLink()),
+			zap.String("lotus_name", ownerRef.Name))
+		return
+	}
+	c.logger.Info("will enqueue new lotus because of an object change",
+		zap.String("object_self_link", object.GetSelfLink()),
+		zap.String("object_name", object.GetName()))
+	c.enqueueLotus(lotus)
+}
+
+func (c *Controller) updateLotusStatus(lotus *lotusv1beta1.Lotus, phase lotusv1beta1.LotusPhase) error {
+	lotus = copyWithNewStatus(lotus, phase)
+	_, err := c.lotusclientset.LotusV1beta1().Lotuses(lotus.Namespace).Update(lotus)
+	return err
+}
+
+func copyWithNewStatus(lotus *lotusv1beta1.Lotus, phase lotusv1beta1.LotusPhase) *lotusv1beta1.Lotus {
+	lotusCopy := lotus.DeepCopy()
+	prev := lotusCopy.Status.Phase
+	if prev != phase {
+		now := metav1.Now()
+		switch phase {
+		case lotusv1beta1.LotusPreparing:
+			lotusCopy.Status.PreparerStartTime = &now
+		case lotusv1beta1.LotusRunning:
+			lotusCopy.Status.WorkerStartTime = &now
+		case lotusv1beta1.LotusCleaning:
+			fallthrough
+		case lotusv1beta1.LotusFailureCleaning:
+			lotusCopy.Status.CleanerStartTime = &now
+		}
+		switch prev {
+		case lotusv1beta1.LotusPreparing:
+			lotusCopy.Status.PreparerCompletionTime = &now
+		case lotusv1beta1.LotusRunning:
+			lotusCopy.Status.WorkerCompletionTime = &now
+		case lotusv1beta1.LotusCleaning:
+			fallthrough
+		case lotusv1beta1.LotusFailureCleaning:
+			lotusCopy.Status.CleanerCompletionTime = &now
+		}
+	}
+	lotusCopy.Status.Phase = phase
+	return lotusCopy
+}
+
+func (c *Controller) ensureStaticResources() error {
+	controllerDeployment, err := c.kubeClient.GetDeployment("lotus-controller", c.namespace)
+	if err != nil {
+		c.logger.Error("failed to get controller deployment", zap.Error(err))
+		return err
+	}
+	owners := []metav1.OwnerReference{
+		*metav1.NewControllerRef(controllerDeployment, schema.GroupVersionKind{
+			Group:   appsv1.SchemeGroupVersion.Group,
+			Version: appsv1.SchemeGroupVersion.Version,
+			Kind:    "Deployment",
+		}),
+	}
+
+	f := resource.NewStaticResourceFactory(c.namespace, c.release, c.configFile, owners)
+	thanosPeerService, err := f.NewThanosPeerService()
+	if err != nil {
+		return err
+	}
+	if err := c.kubeClient.ApplyService(f.ThanosPeerName(), c.namespace, thanosPeerService); err != nil {
+		return err
+	}
+
+	cfg, err := config.FromFile(c.configFile)
+	if err != nil {
+		return err
+	}
+	if cfg.TimeSeriesStorage != nil {
+		timeSeriesStoreSecret, err := f.NewTimeSeriesStoreConfigSecret()
+		if err != nil {
+			return err
+		}
+		if err := c.kubeClient.ApplySecret(f.TimeSeriesStoreConfigSecretName(), c.namespace, timeSeriesStoreSecret); err != nil {
+			return err
+		}
+		thanosStore, err := f.NewThanosStoreStatefulSet()
+		if err != nil {
+			return err
+		}
+		if err := c.kubeClient.ApplyStatefulSet(f.ThanosStoreName(), c.namespace, thanosStore); err != nil {
+			return err
+		}
+	}
+
+	thanosQueryDeployment, err := f.NewThanosQueryDeployment()
+	if err != nil {
+		return err
+	}
+	if err := c.kubeClient.ApplyDeployment(f.ThanosQueryName(), c.namespace, thanosQueryDeployment); err != nil {
+		return err
+	}
+	thanosQueryService, err := f.NewThanosQueryService()
+	if err != nil {
+		return err
+	}
+	return c.kubeClient.ApplyService(f.ThanosQueryName(), c.namespace, thanosQueryService)
+}
diff --git a/pkg/app/lotus/controller/controller_test.go b/pkg/app/lotus/controller/controller_test.go
new file mode 100644
index 0000000..b0b429f
--- /dev/null
+++ b/pkg/app/lotus/controller/controller_test.go
@@ -0,0 +1 @@
+package controller
diff --git a/pkg/app/lotus/datasource/BUILD.bazel b/pkg/app/lotus/datasource/BUILD.bazel
new file mode 100644
index 0000000..cbf3329
--- /dev/null
+++ b/pkg/app/lotus/datasource/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["datasource.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/datasource",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/datasource/datasource.go b/pkg/app/lotus/datasource/datasource.go
new file mode 100644
index 0000000..57cd007
--- /dev/null
+++ b/pkg/app/lotus/datasource/datasource.go
@@ -0,0 +1,56 @@
+package datasource
+
+import (
+	"context"
+	"time"
+
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+)
+
+type Builder interface {
+	Build(ds *config.DataSource, opts BuildOptions) (DataSource, error)
+}
+
+type BuildOptions struct {
+	Logger *zap.Logger
+}
+
+func (o BuildOptions) NamedLogger(name string) *zap.Logger {
+	if o.Logger != nil {
+		return o.Logger.Named(name)
+	}
+	return zap.NewNop().Named(name)
+}
+
+type DataSource interface {
+	Querier
+	Checker
+}
+
+type Querier interface {
+	Query(ctx context.Context, query string, ts time.Time) ([]*Sample, error)
+	CollectSummary(ctx context.Context, ts time.Time) (*model.MetricsSummary, error)
+}
+
+type Sample struct {
+	Labels    map[string]string
+	Value     float64
+	Timestamp time.Time
+}
+
+type Checker interface {
+	Check(ctx context.Context, checks []Check) (*CheckResult, error)
+}
+
+type Check struct {
+	Name string
+	Expr string
+	For  string
+}
+
+type CheckResult struct {
+	Actives []string
+}
diff --git a/pkg/app/lotus/datasource/prometheus/BUILD.bazel b/pkg/app/lotus/datasource/prometheus/BUILD.bazel
new file mode 100644
index 0000000..e945d80
--- /dev/null
+++ b/pkg/app/lotus/datasource/prometheus/BUILD.bazel
@@ -0,0 +1,30 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "builder.go",
+        "prometheus.go",
+        "query.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/datasource/prometheus",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/datasource:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/metrics/grpcmetrics:go_default_library",
+        "//pkg/metrics/httpmetrics:go_default_library",
+        "@com_github_prometheus_client_golang//api:go_default_library",
+        "@com_github_prometheus_client_golang//api/prometheus/v1:go_default_library",
+        "@com_github_prometheus_common//model:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["prometheus_test.go"],
+    embed = [":go_default_library"],
+)
diff --git a/pkg/app/lotus/datasource/prometheus/builder.go b/pkg/app/lotus/datasource/prometheus/builder.go
new file mode 100644
index 0000000..efa0485
--- /dev/null
+++ b/pkg/app/lotus/datasource/prometheus/builder.go
@@ -0,0 +1,35 @@
+package prometheus
+
+import (
+	"fmt"
+
+	promapi "github.com/prometheus/client_golang/api"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/datasource"
+)
+
+type builder struct {
+}
+
+func NewBuilder() datasource.Builder {
+	return &builder{}
+}
+
+func (b *builder) Build(ds *config.DataSource, opts datasource.BuildOptions) (datasource.DataSource, error) {
+	configs, ok := ds.Type.(*config.DataSource_Prometheus)
+	if !ok {
+		return nil, fmt.Errorf("wrong datasource type for prometheus: %T", ds.Type)
+	}
+	cli, err := promapi.NewClient(promapi.Config{
+		Address: configs.Prometheus.Address,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &prometheus{
+		api:    promv1.NewAPI(cli),
+		logger: opts.NamedLogger("prometheus-datasource"),
+	}, nil
+}
diff --git a/pkg/app/lotus/datasource/prometheus/prometheus.go b/pkg/app/lotus/datasource/prometheus/prometheus.go
new file mode 100644
index 0000000..1ee7523
--- /dev/null
+++ b/pkg/app/lotus/datasource/prometheus/prometheus.go
@@ -0,0 +1,288 @@
+package prometheus
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	prommodel "github.com/prometheus/common/model"
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/datasource"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/metrics/grpcmetrics"
+	"github.com/nghialv/lotus/pkg/metrics/httpmetrics"
+)
+
+const (
+	AlertNameLabel   = prommodel.AlertNameLabel
+	AlertStateLabel  = "alertstate"
+	AlertStateFiring = "firing"
+)
+
+type prometheus struct {
+	api    promv1.API
+	logger *zap.Logger
+}
+
+func (p *prometheus) Query(ctx context.Context, query string, ts time.Time) ([]*datasource.Sample, error) {
+	v, err := p.api.Query(ctx, query, ts)
+	if err != nil {
+		return nil, err
+	}
+	vector, ok := v.(prommodel.Vector)
+	if !ok {
+		return nil, fmt.Errorf("unsupported value type: %s, %v", v.Type(), v)
+	}
+	return vectorToSamples(vector), nil
+}
+
+func vectorToSamples(vector prommodel.Vector) []*datasource.Sample {
+	samples := make([]*datasource.Sample, 0, len(vector))
+	for _, s := range vector {
+		sample := &datasource.Sample{
+			Labels: make(map[string]string, len(s.Metric)),
+		}
+		for k, v := range s.Metric {
+			sample.Labels[string(k)] = string(v)
+		}
+		sample.Value = float64(s.Value)
+		sample.Timestamp = s.Timestamp.Time()
+		samples = append(samples, sample)
+	}
+	return samples
+}
+
+func (p *prometheus) Check(ctx context.Context, checks []datasource.Check) (*datasource.CheckResult, error) {
+	var ts time.Time
+	v, err := p.api.Query(ctx, "ALERTS", ts)
+	if err != nil {
+		p.logger.Error("failed to run query to get alerts", zap.Error(err))
+		return nil, err
+	}
+	vector, ok := v.(prommodel.Vector)
+	if !ok {
+		p.logger.Error("unsupported value type", zap.Any("value", v))
+		return nil, fmt.Errorf("unsupported value type: %s", v.Type())
+	}
+	p.logger.Debug("extracting actives", zap.Any("vector", vector), zap.Any("checks", checks))
+	return &datasource.CheckResult{
+		Actives: extractActives(vector, checks),
+	}, nil
+}
+
+func extractActives(vector prommodel.Vector, checks []datasource.Check) []string {
+	targets := make(map[string]struct{}, len(checks))
+	for _, check := range checks {
+		targets[check.Name] = struct{}{}
+	}
+	actives := make(map[string]struct{})
+	for _, sample := range vector {
+		if sample.Value == 0 {
+			continue
+		}
+		name, ok := sample.Metric[AlertNameLabel]
+		if !ok {
+			continue
+		}
+		if _, ok := targets[string(name)]; !ok {
+			continue
+		}
+		state, ok := sample.Metric[AlertStateLabel]
+		if !ok {
+			continue
+		}
+		if state != AlertStateFiring {
+			continue
+		}
+		actives[string(name)] = struct{}{}
+	}
+	list := make([]string, 0, len(actives))
+	for k, _ := range actives {
+		list = append(list, k)
+	}
+	return list
+}
+
+func (p *prometheus) CollectSummary(ctx context.Context, ts time.Time) (*model.MetricsSummary, error) {
+	grpcByMethod, err := p.collectGRPCByMethod(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+	httpByPath, err := p.collectHTTPByPath(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+	grpcAll, err := p.multiQuery(ctx, map[string]string{
+		model.GRPCRPCsKey:              grpcRPCTotalQuery,
+		model.GRPCFailurePercentageKey: grpcFailurePercentageQuery,
+		model.GRPCLatencyAvgKey:        grpcLatencyAvgQuery,
+		model.GRPCSentBytesAvgKey:      grpcSentBytesAvgQuery,
+		model.GRPCReceivedBytesAvgKey:  grpcReceivedBytesAvgQuery,
+	}, ts)
+	if err != nil {
+		return nil, err
+	}
+	httpAll, err := p.multiQuery(ctx, map[string]string{
+		model.HTTPRequestsKey:          httpRequestTotalQuery,
+		model.HTTPFailurePercentageKey: httpFailurePercentageQuery,
+		model.HTTPLatencyAvgKey:        httpLatencyAvgQuery,
+		model.HTTPSentBytesAvgKey:      httpSentBytesAvgQuery,
+		model.HTTPReceivedBytesAvgKey:  httpReceivedBytesAvgQuery,
+	}, ts)
+	if err != nil {
+		return nil, err
+	}
+	summary := &model.MetricsSummary{
+		GRPCRPCTotal:          grpcAll[model.GRPCRPCsKey],
+		GRPCFailurePercentage: grpcAll[model.GRPCFailurePercentageKey],
+		GRPCAll:               grpcAll,
+		GRPCByMethod:          grpcByMethod,
+		HTTPRequestTotal:      httpAll[model.HTTPRequestsKey],
+		HTTPFailurePercentage: httpAll[model.HTTPFailurePercentageKey],
+		HTTPAll:               httpAll,
+		HTTPByPath:            httpByPath,
+	}
+	queries := []struct {
+		Query  string
+		Target *float64
+	}{
+		{
+			Query:  vuStartedTotalQuery,
+			Target: &summary.VirtualUserStartedTotal,
+		},
+		{
+			Query:  vuFailedTotalQuery,
+			Target: &summary.VirtualUserFailedTotal,
+		},
+	}
+	for i := range queries {
+		value, err := p.queryOne(ctx, queries[i].Query, ts)
+		if err != nil {
+			return nil, err
+		}
+		*queries[i].Target = value
+	}
+	return summary, nil
+}
+
+func (p *prometheus) collectGRPCByMethod(ctx context.Context, ts time.Time) (map[string]model.ValueByLabel, error) {
+	result := make(map[string]model.ValueByLabel)
+	queries := map[string]string{
+		model.GRPCRPCsKey:              grpcRPCsByMethodQuery,
+		model.GRPCFailurePercentageKey: grpcFailurePercentageByMethodQuery,
+		model.GRPCLatencyAvgKey:        grpcLatencyAvgByMethodQuery,
+		model.GRPCSentBytesAvgKey:      grpcSentBytesAvgByMethodQuery,
+		model.GRPCReceivedBytesAvgKey:  grpcReceivedBytesAvgByMethodQuery,
+	}
+	for name, query := range queries {
+		values, err := p.queryByLabel(
+			ctx,
+			func(labels map[string]string) (string, bool) {
+				value, ok := labels[grpcmetrics.KeyClientMethod.Name()]
+				return value, ok
+			},
+			query,
+			ts,
+		)
+		if err != nil {
+			return nil, err
+		}
+		for lv, v := range values {
+			if _, ok := result[lv]; !ok {
+				result[lv] = make(map[string]float64)
+			}
+			result[lv][name] = v
+		}
+	}
+	return result, nil
+}
+
+func (p *prometheus) collectHTTPByPath(ctx context.Context, ts time.Time) (map[string]model.ValueByLabel, error) {
+	result := make(map[string]model.ValueByLabel)
+	queries := map[string]string{
+		model.HTTPRequestsKey:          httpRequestsByPathQuery,
+		model.HTTPFailurePercentageKey: httpFailurePercentageByPathQuery,
+		model.HTTPLatencyAvgKey:        httpLatencyAvgByPathQuery,
+		model.HTTPSentBytesAvgKey:      httpSentBytesAvgByPathQuery,
+		model.HTTPReceivedBytesAvgKey:  httpReceivedBytesAvgByPathQuery,
+	}
+	for name, query := range queries {
+		values, err := p.queryByLabel(
+			ctx,
+			func(labels map[string]string) (string, bool) {
+				host, ok := labels[httpmetrics.KeyClientHost.Name()]
+				if !ok {
+					return "", false
+				}
+				route, ok := labels[httpmetrics.KeyClientRoute.Name()]
+				if !ok {
+					return "", false
+				}
+				method, ok := labels[httpmetrics.KeyClientMethod.Name()]
+				if !ok {
+					return "", false
+				}
+				return fmt.Sprintf("%s/%s/%s", method, host, strings.TrimLeft(route, "/")), true
+			},
+			query,
+			ts,
+		)
+		if err != nil {
+			return nil, err
+		}
+		for lv, v := range values {
+			if _, ok := result[lv]; !ok {
+				result[lv] = make(map[string]float64)
+			}
+			result[lv][name] = v
+		}
+	}
+	return result, nil
+}
+
+func (p *prometheus) queryOne(ctx context.Context, query string, ts time.Time) (float64, error) {
+	samples, err := p.Query(ctx, query, ts)
+	if err != nil {
+		return 0, err
+	}
+	if len(samples) > 1 {
+		return 0, fmt.Errorf("response must contain only one sample")
+	}
+	if len(samples) == 0 {
+		return model.NoDataValue, nil
+	}
+	return samples[0].Value, nil
+}
+
+type labelsToKey func(labels map[string]string) (string, bool)
+
+func (p *prometheus) queryByLabel(ctx context.Context, toKey labelsToKey, query string, ts time.Time) (map[string]float64, error) {
+	values := make(map[string]float64)
+	samples, err := p.Query(ctx, query, ts)
+	if err != nil {
+		return nil, err
+	}
+	for _, sample := range samples {
+		key, ok := toKey(sample.Labels)
+		if !ok {
+			continue
+		}
+		values[key] = sample.Value
+	}
+	return values, nil
+}
+
+func (p *prometheus) multiQuery(ctx context.Context, queries map[string]string, ts time.Time) (map[string]float64, error) {
+	values := make(map[string]float64, len(queries))
+	for key, query := range queries {
+		value, err := p.queryOne(ctx, query, ts)
+		if err != nil {
+			return nil, err
+		}
+		values[key] = value
+	}
+	return values, nil
+}
diff --git a/pkg/app/lotus/datasource/prometheus/prometheus_test.go b/pkg/app/lotus/datasource/prometheus/prometheus_test.go
new file mode 100644
index 0000000..9aae1bd
--- /dev/null
+++ b/pkg/app/lotus/datasource/prometheus/prometheus_test.go
@@ -0,0 +1,11 @@
+package prometheus
+
+import "testing"
+
+func TestVectorToSamples(t *testing.T) {
+
+}
+
+func TestExtractActives(t *testing.T) {
+
+}
diff --git a/pkg/app/lotus/datasource/prometheus/query.go b/pkg/app/lotus/datasource/prometheus/query.go
new file mode 100644
index 0000000..64b7cec
--- /dev/null
+++ b/pkg/app/lotus/datasource/prometheus/query.go
@@ -0,0 +1,70 @@
+package prometheus
+
+const (
+	// VirtualUser Queries
+	vuStartedTotalQuery = `sum(max_over_time(lotus_virtual_user_count{virtual_user_status="started"}[1h]))`
+
+	vuFailedTotalQuery = `sum(max_over_time(lotus_virtual_user_count{virtual_user_status="failed"}[1h]))`
+
+	// GRPC Queries
+	grpcRPCTotalQuery = `sum(max_over_time(lotus_grpc_client_completed_rpcs[1h]))`
+
+	grpcFailurePercentageQuery = `100 * sum(max_over_time(lotus_grpc_client_completed_rpcs{grpc_client_status!~"OK|NOT_FOUND"}[1h])) /
+		sum(max_over_time(lotus_grpc_client_completed_rpcs[1h]))`
+
+	grpcLatencyAvgQuery = `sum(max_over_time(lotus_grpc_client_roundtrip_latency_sum[1h])) /
+		sum(max_over_time(lotus_grpc_client_roundtrip_latency_count[1h]))`
+
+	grpcSentBytesAvgQuery = `sum(max_over_time(lotus_grpc_client_sent_bytes_per_rpc_sum[1h])) /
+		sum(max_over_time(lotus_grpc_client_sent_bytes_per_rpc_count[1h]))`
+
+	grpcReceivedBytesAvgQuery = `sum(max_over_time(lotus_grpc_client_received_bytes_per_rpc_sum[1h])) /
+		sum(max_over_time(lotus_grpc_client_received_bytes_per_rpc_count[1h]))`
+
+	// GRPCByMethod Queries
+	grpcMethodLabel = "grpc_client_method"
+
+	grpcRPCsByMethodQuery = `sum by(grpc_client_method) (max_over_time(lotus_grpc_client_completed_rpcs[1h]))`
+
+	grpcFailurePercentageByMethodQuery = `100 * sum by(grpc_client_method) (max_over_time(lotus_grpc_client_completed_rpcs{grpc_client_status!~"OK|NOT_FOUND"}[1h])) /
+		sum by(grpc_client_method) (max_over_time(lotus_grpc_client_completed_rpcs[1h]))`
+
+	grpcLatencyAvgByMethodQuery = `sum by(grpc_client_method) (max_over_time(lotus_grpc_client_roundtrip_latency_sum[1h])) /
+		sum by(grpc_client_method) (max_over_time(lotus_grpc_client_roundtrip_latency_count[1h]))`
+
+	grpcSentBytesAvgByMethodQuery = `sum by(grpc_client_method) (max_over_time(lotus_grpc_client_sent_bytes_per_rpc_sum[1h])) /
+		sum by(grpc_client_method) (max_over_time(lotus_grpc_client_sent_bytes_per_rpc_count[1h]))`
+
+	grpcReceivedBytesAvgByMethodQuery = `sum by(grpc_client_method) (max_over_time(lotus_grpc_client_received_bytes_per_rpc_sum[1h])) /
+		sum by(grpc_client_method) (max_over_time(lotus_grpc_client_received_bytes_per_rpc_count[1h]))`
+
+	// HTTP Queries
+	httpRequestTotalQuery = `sum(max_over_time(lotus_http_client_completed_count[1h]))`
+
+	httpFailurePercentageQuery = `100 * sum(max_over_time(lotus_http_client_completed_count{http_client_status=~"5.."}[1h])) /
+		sum(max_over_time(lotus_http_client_completed_count[1h]))`
+
+	httpLatencyAvgQuery = `sum(max_over_time(lotus_http_client_roundtrip_latency_sum[1h])) /
+		sum(max_over_time(lotus_http_client_roundtrip_latency_count[1h]))`
+
+	httpSentBytesAvgQuery = `sum(max_over_time(lotus_http_client_sent_bytes_sum[1h])) /
+		sum(max_over_time(lotus_http_client_sent_bytes_count[1h]))`
+
+	httpReceivedBytesAvgQuery = `sum(max_over_time(lotus_http_client_received_bytes_sum[1h])) /
+		sum(max_over_time(lotus_http_client_received_bytes_count[1h]))`
+
+	// HTTPByPath Queries
+	httpRequestsByPathQuery = `sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_completed_count[1h]))`
+
+	httpFailurePercentageByPathQuery = `100 * sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_completed_count{http_client_status=~"5.."}[1h])) /
+		sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_completed_count[1h]))`
+
+	httpLatencyAvgByPathQuery = `sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_roundtrip_latency_sum[1h])) /
+		sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_roundtrip_latency_count[1h]))`
+
+	httpSentBytesAvgByPathQuery = `sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_sent_bytes_sum[1h])) /
+		sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_sent_bytes_count[1h]))`
+
+	httpReceivedBytesAvgByPathQuery = `sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_received_bytes_sum[1h])) /
+		sum by(http_client_host,http_client_route,http_client_method) (max_over_time(lotus_http_client_received_bytes_count[1h]))`
+)
diff --git a/pkg/app/lotus/datasource/registry/BUILD.bazel b/pkg/app/lotus/datasource/registry/BUILD.bazel
new file mode 100644
index 0000000..f217796
--- /dev/null
+++ b/pkg/app/lotus/datasource/registry/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["registry.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/datasource/registry",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/datasource:go_default_library",
+        "//pkg/app/lotus/datasource/prometheus:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/datasource/registry/registry.go b/pkg/app/lotus/datasource/registry/registry.go
new file mode 100644
index 0000000..3cb1cb3
--- /dev/null
+++ b/pkg/app/lotus/datasource/registry/registry.go
@@ -0,0 +1,43 @@
+package registry
+
+import (
+	"fmt"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/datasource"
+	"github.com/nghialv/lotus/pkg/app/lotus/datasource/prometheus"
+)
+
+var defaultRegistry = New()
+
+func init() {
+	defaultRegistry.Register(config.DataSource_PROMETHEUS, prometheus.NewBuilder())
+}
+
+func Default() *registry {
+	return defaultRegistry
+}
+
+type registry struct {
+	builders map[config.DataSource_Type]datasource.Builder
+}
+
+func New() *registry {
+	return &registry{
+		builders: make(map[config.DataSource_Type]datasource.Builder),
+	}
+}
+
+func (r *registry) Register(dst config.DataSource_Type, b datasource.Builder) {
+	if r.builders[dst] != nil {
+		panic(fmt.Sprintf("duplicate builder registered: %v", dst))
+	}
+	r.builders[dst] = b
+}
+
+func (r *registry) Get(dst config.DataSource_Type) (datasource.Builder, error) {
+	if b, ok := r.builders[dst]; ok {
+		return b, nil
+	}
+	return nil, fmt.Errorf("unknown builder: %v", dst)
+}
diff --git a/pkg/app/lotus/kubeclient/BUILD.bazel b/pkg/app/lotus/kubeclient/BUILD.bazel
new file mode 100644
index 0000000..6d22d59
--- /dev/null
+++ b/pkg/app/lotus/kubeclient/BUILD.bazel
@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["kubeclient.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/kubeclient",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@io_k8s_api//apps/v1:go_default_library",
+        "@io_k8s_api//batch/v1:go_default_library",
+        "@io_k8s_api//core/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_client_go//kubernetes:go_default_library",
+        "@io_k8s_client_go//listers/batch/v1:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["kubeclient_test.go"],
+    embed = [":go_default_library"],
+)
diff --git a/pkg/app/lotus/kubeclient/kubeclient.go b/pkg/app/lotus/kubeclient/kubeclient.go
new file mode 100644
index 0000000..d9425eb
--- /dev/null
+++ b/pkg/app/lotus/kubeclient/kubeclient.go
@@ -0,0 +1,158 @@
+package kubeclient
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	batchlisters "k8s.io/client-go/listers/batch/v1"
+)
+
+type KubeClient interface {
+	EnsureDeployment(name, namespace string, factory func() (*appsv1.Deployment, error)) (*appsv1.Deployment, error)
+	EnsurePod(name, namespace string, factory func() (*corev1.Pod, error)) (*corev1.Pod, error)
+	EnsureService(name, namespace string, factory func() (*corev1.Service, error)) (*corev1.Service, error)
+	EnsureConfigMap(name, namespace string, factory func() (*corev1.ConfigMap, error)) (*corev1.ConfigMap, error)
+	EnsureJob(name, namespace string, factory func() (*v1.Job, error)) (*v1.Job, error)
+
+	ApplyStatefulSet(name, namespace string, s *appsv1.StatefulSet) error
+	ApplyService(name, namespace string, s *corev1.Service) error
+	ApplyDeployment(name, namespace string, d *appsv1.Deployment) error
+	ApplySecret(name, namespace string, s *corev1.Secret) error
+	GetDeployment(name, namespace string) (*appsv1.Deployment, error)
+	DeleteDeployment(name, namespace string) error
+}
+
+func New(kubeClientSet kubernetes.Interface, jobsLister batchlisters.JobLister) KubeClient {
+	return &kubeclient{
+		kubeClientSet: kubeClientSet,
+		jobsLister:    jobsLister,
+	}
+}
+
+type kubeclient struct {
+	kubeClientSet kubernetes.Interface
+	jobsLister    batchlisters.JobLister
+}
+
+func (c *kubeclient) EnsureDeployment(name, namespace string, factory func() (*appsv1.Deployment, error)) (*appsv1.Deployment, error) {
+	deployment, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{})
+	if !errors.IsNotFound(err) {
+		return deployment, err
+	}
+	deployment, err = factory()
+	if err != nil {
+		return nil, err
+	}
+	return c.kubeClientSet.AppsV1().Deployments(namespace).Create(deployment)
+}
+
+func (c *kubeclient) EnsurePod(name, namespace string, factory func() (*corev1.Pod, error)) (*corev1.Pod, error) {
+	pod, err := c.kubeClientSet.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
+	if !errors.IsNotFound(err) {
+		return pod, err
+	}
+	pod, err = factory()
+	if err != nil {
+		return nil, err
+	}
+	return c.kubeClientSet.CoreV1().Pods(namespace).Create(pod)
+}
+
+func (c *kubeclient) EnsureService(name, namespace string, factory func() (*corev1.Service, error)) (*corev1.Service, error) {
+	service, err := c.kubeClientSet.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
+	if !errors.IsNotFound(err) {
+		return service, err
+	}
+	service, err = factory()
+	if err != nil {
+		return nil, err
+	}
+	return c.kubeClientSet.CoreV1().Services(namespace).Create(service)
+}
+
+func (c *kubeclient) EnsureConfigMap(name, namespace string, factory func() (*corev1.ConfigMap, error)) (*corev1.ConfigMap, error) {
+	configmap, err := c.kubeClientSet.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
+	if !errors.IsNotFound(err) {
+		return configmap, err
+	}
+	configmap, err = factory()
+	if err != nil {
+		return nil, err
+	}
+	return c.kubeClientSet.CoreV1().ConfigMaps(namespace).Create(configmap)
+}
+
+func (c *kubeclient) EnsureJob(name, namespace string, factory func() (*v1.Job, error)) (*v1.Job, error) {
+	job, err := c.jobsLister.Jobs(namespace).Get(name)
+	if !errors.IsNotFound(err) {
+		return job, err
+	}
+	job, err = factory()
+	if err != nil {
+		return nil, err
+	}
+	return c.kubeClientSet.BatchV1().Jobs(namespace).Create(job)
+}
+
+func (c *kubeclient) ApplyStatefulSet(name, namespace string, s *appsv1.StatefulSet) error {
+	_, err := c.kubeClientSet.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		_, err = c.kubeClientSet.AppsV1().StatefulSets(namespace).Create(s)
+		return err
+	}
+	if err == nil {
+		_, err = c.kubeClientSet.AppsV1().StatefulSets(namespace).Update(s)
+	}
+	return err
+}
+
+func (c *kubeclient) ApplyService(name, namespace string, s *corev1.Service) error {
+	_, err := c.kubeClientSet.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		_, err = c.kubeClientSet.CoreV1().Services(namespace).Create(s)
+		return err
+	}
+	if err == nil {
+		_, err = c.kubeClientSet.CoreV1().Services(namespace).Update(s)
+	}
+	return err
+}
+
+func (c *kubeclient) ApplyDeployment(name, namespace string, d *appsv1.Deployment) error {
+	_, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		_, err = c.kubeClientSet.AppsV1().Deployments(namespace).Create(d)
+		return err
+	}
+	if err == nil {
+		_, err = c.kubeClientSet.AppsV1().Deployments(namespace).Update(d)
+	}
+	return err
+}
+
+func (c *kubeclient) ApplySecret(name, namespace string, s *corev1.Secret) error {
+	_, err := c.kubeClientSet.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		_, err = c.kubeClientSet.CoreV1().Secrets(namespace).Create(s)
+		return err
+	}
+	if err == nil {
+		_, err = c.kubeClientSet.CoreV1().Secrets(namespace).Update(s)
+	}
+	return err
+}
+
+func (c *kubeclient) GetDeployment(name, namespace string) (*appsv1.Deployment, error) {
+	return c.kubeClientSet.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{})
+}
+
+func (c *kubeclient) DeleteDeployment(name, namespace string) error {
+	err := c.kubeClientSet.AppsV1().Deployments(namespace).Delete(name, nil)
+	if err == nil || errors.IsNotFound(err) {
+		return nil
+	}
+	return err
+}
diff --git a/pkg/app/lotus/kubeclient/kubeclient_test.go b/pkg/app/lotus/kubeclient/kubeclient_test.go
new file mode 100644
index 0000000..300b454
--- /dev/null
+++ b/pkg/app/lotus/kubeclient/kubeclient_test.go
@@ -0,0 +1 @@
+package kubeclient
diff --git a/pkg/app/lotus/model/BUILD.bazel b/pkg/app/lotus/model/BUILD.bazel
new file mode 100644
index 0000000..2ec5838
--- /dev/null
+++ b/pkg/app/lotus/model/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "lotus.go",
+        "metrics_summary.go",
+        "render.go",
+        "result.go",
+        "templates.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/model",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["render_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "@com_github_stretchr_testify//assert:go_default_library",
+        "@com_github_stretchr_testify//require:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/model/lotus.go b/pkg/app/lotus/model/lotus.go
new file mode 100644
index 0000000..56f593a
--- /dev/null
+++ b/pkg/app/lotus/model/lotus.go
@@ -0,0 +1,18 @@
+package model
+
+import (
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+const (
+	LotusKind = "Lotus"
+)
+
+var (
+	ControllerKind = schema.GroupVersionKind{
+		Group:   lotusv1beta1.SchemeGroupVersion.Group,
+		Version: lotusv1beta1.SchemeGroupVersion.Version,
+		Kind:    LotusKind,
+	}
+)
diff --git a/pkg/app/lotus/model/metrics_summary.go b/pkg/app/lotus/model/metrics_summary.go
new file mode 100644
index 0000000..291aef0
--- /dev/null
+++ b/pkg/app/lotus/model/metrics_summary.go
@@ -0,0 +1,36 @@
+package model
+
+const (
+	NoDataValue float64 = -1
+)
+
+type MetricsSummary struct {
+	GRPCRPCTotal          float64
+	GRPCFailurePercentage float64
+	GRPCAll               ValueByLabel
+	GRPCByMethod          map[string]ValueByLabel
+
+	HTTPRequestTotal      float64
+	HTTPFailurePercentage float64
+	HTTPAll               ValueByLabel
+	HTTPByPath            map[string]ValueByLabel
+
+	VirtualUserStartedTotal float64
+	VirtualUserFailedTotal  float64
+}
+
+type ValueByLabel map[string]float64
+
+const (
+	GRPCRPCsKey              = "RPCs"
+	GRPCFailurePercentageKey = "FailurePercentage"
+	GRPCLatencyAvgKey        = "LatencyAvg"
+	GRPCSentBytesAvgKey      = "SentBytesAvg"
+	GRPCReceivedBytesAvgKey  = "ReceivedBytesAvg"
+
+	HTTPRequestsKey          = "Requests"
+	HTTPFailurePercentageKey = "FailurePercentage"
+	HTTPLatencyAvgKey        = "LatencyAvg"
+	HTTPSentBytesAvgKey      = "SentBytesAvg"
+	HTTPReceivedBytesAvgKey  = "ReceivedBytesAvg"
+)
diff --git a/pkg/app/lotus/model/render.go b/pkg/app/lotus/model/render.go
new file mode 100644
index 0000000..9887445
--- /dev/null
+++ b/pkg/app/lotus/model/render.go
@@ -0,0 +1,184 @@
+package model
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"text/template"
+	"time"
+)
+
+type RenderFormat string
+
+const (
+	RenderFormatMarkdown RenderFormat = "markdown"
+	RenderFormatText                  = "text"
+	RenderFormatJson                  = "json"
+
+	noDataMark = "--"
+	timeFormat = "15:04:05 2006-01-02"
+)
+
+type valueByLabelList struct {
+	name   string
+	values ValueByLabel
+}
+
+func renderTemplate(result *Result, tpl string) ([]byte, error) {
+	funcMap := template.FuncMap{
+		"formatValue":        formatValue,
+		"formatTime":         formatTime,
+		"formatGRPCByMethod": formatGRPCByMethod,
+		"formatHTTPByPath":   formatHTTPByPath,
+	}
+	template, err := template.New("result").Funcs(funcMap).Parse(tpl)
+	if err != nil {
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = template.Execute(&buffer, result)
+	if err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func formatGRPCByMethod(data map[string]ValueByLabel, all ValueByLabel) string {
+	keys := []struct {
+		Desc string
+		Key  string
+	}{
+		{Desc: "RPCs", Key: GRPCRPCsKey},
+		{Desc: `Failure%`, Key: GRPCFailurePercentageKey},
+		{Desc: "Latency(ms)", Key: GRPCLatencyAvgKey},
+		{Desc: "SentBytes", Key: GRPCSentBytesAvgKey},
+		{Desc: "RecvBytes", Key: GRPCReceivedBytesAvgKey},
+	}
+	methodMaxLength := 5
+	for method, _ := range data {
+		if len(method) > methodMaxLength {
+			methodMaxLength = len(method)
+		}
+	}
+	var b bytes.Buffer
+	format := func(key string, values ValueByLabel) string {
+		value, ok := values[key]
+		if !ok {
+			value = NoDataValue
+		}
+		return formatValue(value)
+	}
+	titleFormat := fmt.Sprintf("    %%-%ds  %%-8s  %%-8s  %%-12s  %%-8s  %%-8s\n\n", methodMaxLength)
+	detailFormat := fmt.Sprintf("  - %%-%ds  %%-8s  %%-8s  %%-12s  %%-8s  %%-8s\n", methodMaxLength)
+
+	b.WriteString(fmt.Sprintf(titleFormat, "", keys[0].Desc, keys[1].Desc, keys[2].Desc, keys[3].Desc, keys[4].Desc))
+	rows := make([]valueByLabelList, 0, len(data)+1)
+	for method, values := range data {
+		rows = append(rows, valueByLabelList{
+			name:   method,
+			values: values,
+		})
+	}
+	rows = append(rows, valueByLabelList{
+		name:   "all",
+		values: all,
+	})
+	for _, row := range rows {
+		b.WriteString(fmt.Sprintf(detailFormat,
+			row.name,
+			format(keys[0].Key, row.values),
+			format(keys[1].Key, row.values),
+			format(keys[2].Key, row.values),
+			format(keys[3].Key, row.values),
+			format(keys[4].Key, row.values),
+		))
+	}
+	return b.String()
+}
+
+func formatHTTPByPath(data map[string]ValueByLabel, all ValueByLabel) string {
+	keys := []struct {
+		Desc string
+		Key  string
+	}{
+		{Desc: "Requests", Key: HTTPRequestsKey},
+		{Desc: `Failure%`, Key: HTTPFailurePercentageKey},
+		{Desc: "Latency(ms)", Key: HTTPLatencyAvgKey},
+		{Desc: "SentBytes", Key: HTTPSentBytesAvgKey},
+		{Desc: "RecvBytes", Key: HTTPReceivedBytesAvgKey},
+	}
+	pathMaxLength := 5
+	for path, _ := range data {
+		if len(path) > pathMaxLength {
+			pathMaxLength = len(path)
+		}
+	}
+	var b bytes.Buffer
+	format := func(key string, values ValueByLabel) string {
+		value, ok := values[key]
+		if !ok {
+			value = NoDataValue
+		}
+		return formatValue(value)
+	}
+	titleFormat := fmt.Sprintf("    %%-%ds  %%-8s  %%-8s  %%-12s  %%-8s  %%-8s\n\n", pathMaxLength)
+	detailFormat := fmt.Sprintf("  - %%-%ds  %%-8s  %%-8s  %%-12s  %%-8s  %%-8s\n", pathMaxLength)
+
+	b.WriteString(fmt.Sprintf(titleFormat, "", keys[0].Desc, keys[1].Desc, keys[2].Desc, keys[3].Desc, keys[4].Desc))
+	rows := make([]valueByLabelList, 0, len(data)+1)
+	for path, values := range data {
+		rows = append(rows, valueByLabelList{
+			name:   path,
+			values: values,
+		})
+	}
+	rows = append(rows, valueByLabelList{
+		name:   "all",
+		values: all,
+	})
+	for _, row := range rows {
+		b.WriteString(fmt.Sprintf(detailFormat,
+			row.name,
+			format(keys[0].Key, row.values),
+			format(keys[1].Key, row.values),
+			format(keys[2].Key, row.values),
+			format(keys[3].Key, row.values),
+			format(keys[4].Key, row.values),
+		))
+	}
+	return b.String()
+}
+
+// https://en.wikipedia.org/wiki/Metric_prefix
+func formatValue(v float64) string {
+	if v == NoDataValue {
+		return noDataMark
+	}
+	if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+		return fmt.Sprintf("%.4g", v)
+	}
+	if math.Abs(v) >= 1 {
+		prefix := ""
+		for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+			if math.Abs(v) < 1000 {
+				break
+			}
+			prefix = p
+			v /= 1000
+		}
+		return fmt.Sprintf("%.4g%s", v, prefix)
+	}
+	prefix := ""
+	for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+		if math.Abs(v) >= 1 {
+			break
+		}
+		prefix = p
+		v *= 1000
+	}
+	return fmt.Sprintf("%.4g%s", v, prefix)
+}
+
+func formatTime(t time.Time) string {
+	return t.Format(timeFormat)
+}
diff --git a/pkg/app/lotus/model/render_test.go b/pkg/app/lotus/model/render_test.go
new file mode 100644
index 0000000..4f12062
--- /dev/null
+++ b/pkg/app/lotus/model/render_test.go
@@ -0,0 +1,99 @@
+package model
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRender(t *testing.T) {
+	metricsSummary := &MetricsSummary{
+		GRPCRPCTotal:          25000000,
+		GRPCFailurePercentage: 2.507,
+		GRPCAll: map[string]float64{
+			GRPCRPCsKey:              25000000,
+			GRPCFailurePercentageKey: 1.207,
+			GRPCLatencyAvgKey:        135,
+			GRPCSentBytesAvgKey:      12,
+			GRPCReceivedBytesAvgKey:  245,
+		},
+		GRPCByMethod: map[string]ValueByLabel{
+			"helloworld.Hello": map[string]float64{
+				GRPCRPCsKey:              12500000,
+				GRPCFailurePercentageKey: 1.015,
+				GRPCLatencyAvgKey:        105,
+				GRPCSentBytesAvgKey:      15,
+				GRPCReceivedBytesAvgKey:  8,
+			},
+			"helloworld.Profile": map[string]float64{
+				GRPCRPCsKey:              12500000,
+				GRPCFailurePercentageKey: 1.415,
+				GRPCLatencyAvgKey:        152,
+				GRPCSentBytesAvgKey:      8,
+				GRPCReceivedBytesAvgKey:  256,
+			},
+		},
+		HTTPRequestTotal:      10,
+		HTTPFailurePercentage: 1.05890,
+		HTTPAll: map[string]float64{
+			HTTPRequestsKey:          10,
+			HTTPFailurePercentageKey: 1.05890,
+		},
+		VirtualUserStartedTotal: 1000000,
+		VirtualUserFailedTotal:  0,
+	}
+	testcases := []struct {
+		Result *Result
+	}{
+		{
+			Result: &Result{
+				TestID:            "test-scenario-12345",
+				Status:            TestSucceeded,
+				MetricsSummary:    metricsSummary,
+				StartedTimestamp:  time.Now().Add(-10 * time.Minute),
+				FinishedTimestamp: time.Now(),
+			},
+		},
+	}
+	for _, tc := range testcases {
+		tc.Result.SetGrafanaDashboardURLs("http://localhost:3000")
+		out, err := tc.Result.Render(RenderFormatText)
+		require.NoError(t, err)
+		fmt.Println(string(out))
+	}
+}
+
+func TestFormatValue(t *testing.T) {
+	testcases := []struct {
+		Value    float64
+		Expected string
+	}{
+		{
+			Value:    -1,
+			Expected: noDataMark,
+		},
+		{
+			Value:    0.0,
+			Expected: "0",
+		},
+		{
+			Value:    2.15,
+			Expected: "2.15",
+		},
+		{
+			Value:    12345678,
+			Expected: "12.35M",
+		},
+		{
+			Value:    0.123456,
+			Expected: "123.5m",
+		},
+	}
+	for _, tc := range testcases {
+		out := formatValue(tc.Value)
+		assert.Equal(t, tc.Expected, out)
+	}
+}
diff --git a/pkg/app/lotus/model/result.go b/pkg/app/lotus/model/result.go
new file mode 100644
index 0000000..d5055dc
--- /dev/null
+++ b/pkg/app/lotus/model/result.go
@@ -0,0 +1,53 @@
+package model
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+type TestStatus string
+
+const (
+	TestSucceeded TestStatus = "Succeeded"
+	TestFailed               = "Failed"
+	TestCancelled            = "Cancelled"
+)
+
+type Result struct {
+	TestID                   string
+	Status                   TestStatus
+	MetricsSummary           *MetricsSummary
+	FailureReason            string
+	FailedChecks             []string
+	StartedTimestamp         time.Time
+	FinishedTimestamp        time.Time
+	GrafanaGRPCDashboardsURL string
+	GrafanaHTTPDashboardsURL string
+}
+
+func (r *Result) SetFailed(reason string) {
+	r.Status = TestFailed
+	r.FailureReason = reason
+}
+
+func (r *Result) SetGrafanaDashboardURLs(base string) {
+	base = strings.TrimRight(base, "/")
+	var from int64 = r.StartedTimestamp.Add(-time.Minute).UnixNano() / 1e6
+	var to int64 = r.FinishedTimestamp.Add(time.Minute).UnixNano() / 1e6
+	r.GrafanaGRPCDashboardsURL = fmt.Sprintf("%s/dashboard/db/grpc?from=%d&to=%d&var-testId=%s", base, from, to, r.TestID)
+	r.GrafanaHTTPDashboardsURL = fmt.Sprintf("%s/dashboard/db/http?from=%d&to=%d&var-testId=%s", base, from, to, r.TestID)
+}
+
+func (r *Result) Render(format RenderFormat) ([]byte, error) {
+	switch format {
+	case RenderFormatMarkdown:
+		return renderTemplate(r, markdownTemplate)
+	case RenderFormatText:
+		return renderTemplate(r, textTemplate)
+	case RenderFormatJson:
+		return json.Marshal(r)
+	}
+	return nil, fmt.Errorf("unsupported render format: %s", format)
+}
diff --git a/pkg/app/lotus/model/templates.go b/pkg/app/lotus/model/templates.go
new file mode 100644
index 0000000..75f7be1
--- /dev/null
+++ b/pkg/app/lotus/model/templates.go
@@ -0,0 +1,48 @@
+package model
+
+const (
+	textTemplate = `
+TestID:        {{ .TestID }}
+TestStatus:    {{ .Status }}
+{{- if eq .Status "Failed" }}
+    Reason: {{ .FailureReason }}
+{{- if gt (len .FailedChecks) 0 }}
+    FailedChecks:  {{ .FailedChecks }}
+{{- end }}
+{{- end }}
+Start:         {{ formatTime .StartedTimestamp }}
+End:           {{ formatTime .FinishedTimestamp }}
+
+MetricsSummary:
+
+{{- if .MetricsSummary }}
+
+1. Virtual User
+  - Started:             {{ formatValue .MetricsSummary.VirtualUserStartedTotal }}
+  - Failed:              {{ formatValue .MetricsSummary.VirtualUserFailedTotal }}
+
+2. GRPC
+  - RPCTotal:            {{ formatValue .MetricsSummary.GRPCRPCTotal }}
+  - FailurePercentage:   {{ formatValue .MetricsSummary.GRPCFailurePercentage }}
+
+GroupByMethod:
+{{ formatGRPCByMethod .MetricsSummary.GRPCByMethod .MetricsSummary.GRPCAll }}
+Grafana: {{ .GrafanaGRPCDashboardsURL }}
+
+3. HTTP
+  - RequestTotal:        {{ formatValue .MetricsSummary.HTTPRequestTotal }}
+  - FailurePercentage:   {{ formatValue .MetricsSummary.HTTPFailurePercentage }}
+
+GroupByPath:
+{{ formatHTTPByPath .MetricsSummary.HTTPByPath .MetricsSummary.HTTPAll }}
+Grafana: {{ .GrafanaHTTPDashboardsURL }}
+{{- else }}
+
+  No data
+{{- end }}
+`
+)
+
+var (
+	markdownTemplate = textTemplate
+)
diff --git a/pkg/app/lotus/reporter/BUILD.bazel b/pkg/app/lotus/reporter/BUILD.bazel
new file mode 100644
index 0000000..b220246
--- /dev/null
+++ b/pkg/app/lotus/reporter/BUILD.bazel
@@ -0,0 +1,25 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["reporter.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "@org_golang_x_sync//errgroup:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["reporter_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/app/lotus/model:go_default_library",
+        "@com_github_stretchr_testify//assert:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/reporter/azure/BUILD.bazel b/pkg/app/lotus/reporter/azure/BUILD.bazel
new file mode 100644
index 0000000..71aabd0
--- /dev/null
+++ b/pkg/app/lotus/reporter/azure/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["azure.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/azure",
+    visibility = ["//visibility:public"],
+)
diff --git a/pkg/app/lotus/reporter/azure/azure.go b/pkg/app/lotus/reporter/azure/azure.go
new file mode 100644
index 0000000..fd6f983
--- /dev/null
+++ b/pkg/app/lotus/reporter/azure/azure.go
@@ -0,0 +1,3 @@
+package azure
+
+//TODO: implementation
diff --git a/pkg/app/lotus/reporter/gcs/BUILD.bazel b/pkg/app/lotus/reporter/gcs/BUILD.bazel
new file mode 100644
index 0000000..220eea3
--- /dev/null
+++ b/pkg/app/lotus/reporter/gcs/BUILD.bazel
@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["gcs.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/gcs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/app/lotus/reporter:go_default_library",
+        "@com_google_cloud_go//storage:go_default_library",
+        "@org_golang_google_api//option:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/reporter/gcs/gcs.go b/pkg/app/lotus/reporter/gcs/gcs.go
new file mode 100644
index 0000000..5e9f4a1
--- /dev/null
+++ b/pkg/app/lotus/reporter/gcs/gcs.go
@@ -0,0 +1,95 @@
+package gcs
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/storage"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter"
+)
+
+type builder struct {
+}
+
+func NewBuilder() reporter.Builder {
+	return &builder{}
+}
+
+func (b *builder) Build(r *config.Receiver, opts reporter.BuildOptions) (reporter.Reporter, error) {
+	configs, ok := r.Type.(*config.Receiver_Gcs)
+	if !ok {
+		return nil, fmt.Errorf("wrong receiver type for gcs: %T", r.Type)
+	}
+	return &gcs{
+		bucket:          configs.Gcs.Bucket,
+		credentialsFile: r.CredentialsFile(configs.Gcs.Credentials.File),
+		logger:          opts.NamedLogger("gcs-reporter"),
+	}, nil
+}
+
+type gcs struct {
+	bucket          string
+	credentialsFile string
+	logger          *zap.Logger
+}
+
+func (g *gcs) Report(ctx context.Context, result *model.Result) (lastErr error) {
+	client, err := storage.NewClient(ctx, option.WithCredentialsFile(g.credentialsFile))
+	if err != nil {
+		g.logger.Error("failed to create gcs storage client", zap.Error(err))
+		lastErr = err
+		return
+	}
+	cases := []struct {
+		format      model.RenderFormat
+		extension   string
+		contentType string
+	}{
+		{
+			format:      model.RenderFormatText,
+			extension:   "txt",
+			contentType: "text/plain",
+		},
+		{
+			format:      model.RenderFormatJson,
+			extension:   "json",
+			contentType: "application/json",
+		},
+	}
+	for _, c := range cases {
+		data, err := result.Render(c.format)
+		if err != nil {
+			g.logger.Error("failed to render result", zap.Error(err))
+			lastErr = err
+			continue
+		}
+		filename := fmt.Sprintf("%s/%s.%s", result.TestID, result.TestID, c.extension)
+		g.logger.Info("writing test result to gcs storage",
+			zap.String("testID", result.TestID),
+			zap.String("filename", filename),
+			zap.String("bucket", g.bucket),
+		)
+		wc := client.Bucket(g.bucket).Object(filename).NewWriter(ctx)
+		wc.ContentType = c.contentType
+		wc.ACL = []storage.ACLRule{{
+			Entity: storage.AllUsers,
+			Role:   storage.RoleReader,
+		}}
+		if _, err := wc.Write(data); err != nil {
+			g.logger.Error("failed to write result", zap.Error(err))
+			lastErr = err
+			continue
+		}
+		if err := wc.Close(); err != nil {
+			g.logger.Error("failed to close writer", zap.Error(err))
+			lastErr = err
+			continue
+		}
+	}
+	return
+}
diff --git a/pkg/app/lotus/reporter/logger/BUILD.bazel b/pkg/app/lotus/reporter/logger/BUILD.bazel
new file mode 100644
index 0000000..266bc83
--- /dev/null
+++ b/pkg/app/lotus/reporter/logger/BUILD.bazel
@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["logger.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/logger",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/app/lotus/reporter:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/reporter/logger/logger.go b/pkg/app/lotus/reporter/logger/logger.go
new file mode 100644
index 0000000..af48cfc
--- /dev/null
+++ b/pkg/app/lotus/reporter/logger/logger.go
@@ -0,0 +1,40 @@
+package logger
+
+import (
+	"context"
+	"fmt"
+
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter"
+)
+
+type builder struct {
+}
+
+func NewBuilder() reporter.Builder {
+	return &builder{}
+}
+
+func (b *builder) Build(r *config.Receiver, opts reporter.BuildOptions) (reporter.Reporter, error) {
+	return &logger{
+		logger: opts.NamedLogger("logger-reporter"),
+	}, nil
+}
+
+type logger struct {
+	logger *zap.Logger
+}
+
+func (p *logger) Report(ctx context.Context, result *model.Result) error {
+	out, err := result.Render(model.RenderFormatText)
+	if err != nil {
+		return err
+	}
+	fmt.Println("=========== TEST RESULT ==========")
+	fmt.Println(string(out))
+	fmt.Println("===========     END     ==========")
+	return nil
+}
diff --git a/pkg/app/lotus/reporter/registry/BUILD.bazel b/pkg/app/lotus/reporter/registry/BUILD.bazel
new file mode 100644
index 0000000..b7cce5c
--- /dev/null
+++ b/pkg/app/lotus/reporter/registry/BUILD.bazel
@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["registry.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/registry",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/reporter:go_default_library",
+        "//pkg/app/lotus/reporter/gcs:go_default_library",
+        "//pkg/app/lotus/reporter/logger:go_default_library",
+        "//pkg/app/lotus/reporter/slack:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/reporter/registry/registry.go b/pkg/app/lotus/reporter/registry/registry.go
new file mode 100644
index 0000000..5d0d96c
--- /dev/null
+++ b/pkg/app/lotus/reporter/registry/registry.go
@@ -0,0 +1,47 @@
+package registry
+
+import (
+	"fmt"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter/gcs"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter/logger"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter/slack"
+)
+
+var defaultRegistry = New()
+
+func init() {
+	defaultRegistry.Register(config.Receiver_LOGGER, logger.NewBuilder())
+	defaultRegistry.Register(config.Receiver_GCS, gcs.NewBuilder())
+	defaultRegistry.Register(config.Receiver_SLACK, slack.NewBuilder())
+}
+
+func Default() *registry {
+	return defaultRegistry
+}
+
+type registry struct {
+	builders map[config.Receiver_Type]reporter.Builder
+}
+
+func New() *registry {
+	return &registry{
+		builders: make(map[config.Receiver_Type]reporter.Builder),
+	}
+}
+
+func (r *registry) Register(rt config.Receiver_Type, b reporter.Builder) {
+	if r.builders[rt] != nil {
+		panic(fmt.Sprintf("duplicate builder registered: %v", rt))
+	}
+	r.builders[rt] = b
+}
+
+func (r *registry) Get(rt config.Receiver_Type) (reporter.Builder, error) {
+	if b, ok := r.builders[rt]; ok {
+		return b, nil
+	}
+	return nil, fmt.Errorf("unknown builder: %v", rt)
+}
diff --git a/pkg/app/lotus/reporter/reporter.go b/pkg/app/lotus/reporter/reporter.go
new file mode 100644
index 0000000..4f24b95
--- /dev/null
+++ b/pkg/app/lotus/reporter/reporter.go
@@ -0,0 +1,65 @@
+package reporter
+
+import (
+	"context"
+
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+)
+
+type Builder interface {
+	Build(r *config.Receiver, opts BuildOptions) (Reporter, error)
+}
+
+type BuildOptions struct {
+	Logger *zap.Logger
+}
+
+func (o BuildOptions) NamedLogger(name string) *zap.Logger {
+	if o.Logger != nil {
+		return o.Logger.Named(name)
+	}
+	return zap.NewNop().Named(name)
+}
+
+type Reporter interface {
+	Report(ctx context.Context, result *model.Result) error
+}
+
+type multiReporter struct {
+	reporters []Reporter
+}
+
+func MultiReporter(reporters ...Reporter) Reporter {
+	all := make([]Reporter, 0, len(reporters))
+	for _, r := range reporters {
+		if mr, ok := r.(*multiReporter); ok {
+			all = append(all, mr.reporters...)
+		} else {
+			all = append(all, r)
+		}
+	}
+	return &multiReporter{
+		reporters: all,
+	}
+}
+
+func (mr *multiReporter) Report(ctx context.Context, result *model.Result) error {
+	if len(mr.reporters) == 0 {
+		return nil
+	}
+	if len(mr.reporters) == 1 {
+		return mr.reporters[0].Report(ctx, result)
+	}
+	g, ctx := errgroup.WithContext(ctx)
+	for i := range mr.reporters {
+		reporter := mr.reporters[i]
+		g.Go(func() error {
+			return reporter.Report(ctx, result)
+		})
+	}
+	return g.Wait()
+}
diff --git a/pkg/app/lotus/reporter/reporter_test.go b/pkg/app/lotus/reporter/reporter_test.go
new file mode 100644
index 0000000..f0245e0
--- /dev/null
+++ b/pkg/app/lotus/reporter/reporter_test.go
@@ -0,0 +1,66 @@
+package reporter
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/stretchr/testify/assert"
+)
+
+type reporterFunc func(ctx context.Context, result *model.Result) error
+
+func (f reporterFunc) Report(ctx context.Context, result *model.Result) error {
+	return f(ctx, result)
+}
+
+func TestMultiReporter(t *testing.T) {
+	var calls int
+	successReporter := reporterFunc(func(ctx context.Context, result *model.Result) error {
+		calls++
+		return nil
+	})
+	failureReporter := reporterFunc(func(ctx context.Context, result *model.Result) error {
+		calls++
+		return errors.New("failed")
+	})
+	testcases := []struct {
+		reporter Reporter
+		result   *model.Result
+		hasError bool
+		calls    int
+	}{
+		{
+			reporter: MultiReporter(successReporter),
+			hasError: false,
+			calls:    1,
+		},
+		{
+			reporter: MultiReporter(successReporter, successReporter),
+			hasError: false,
+			calls:    2,
+		},
+		{
+			reporter: MultiReporter(successReporter, failureReporter),
+			hasError: true,
+			calls:    2,
+		},
+		{
+			reporter: MultiReporter(successReporter, MultiReporter(successReporter, successReporter)),
+			hasError: false,
+			calls:    3,
+		},
+		{
+			reporter: MultiReporter(successReporter, MultiReporter(successReporter, failureReporter)),
+			hasError: true,
+			calls:    3,
+		},
+	}
+	for _, tc := range testcases {
+		calls = 0
+		err := tc.reporter.Report(context.TODO(), tc.result)
+		assert.Equal(t, tc.hasError, err != nil)
+		assert.Equal(t, tc.calls, calls)
+	}
+}
diff --git a/pkg/app/lotus/reporter/s3/BUILD.bazel b/pkg/app/lotus/reporter/s3/BUILD.bazel
new file mode 100644
index 0000000..c5a78b9
--- /dev/null
+++ b/pkg/app/lotus/reporter/s3/BUILD.bazel
@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["s3.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/s3",
+    visibility = ["//visibility:public"],
+)
diff --git a/pkg/app/lotus/reporter/s3/s3.go b/pkg/app/lotus/reporter/s3/s3.go
new file mode 100644
index 0000000..d0ae1f7
--- /dev/null
+++ b/pkg/app/lotus/reporter/s3/s3.go
@@ -0,0 +1,3 @@
+package s3
+
+// TODO: Implementation
diff --git a/pkg/app/lotus/reporter/slack/BUILD.bazel b/pkg/app/lotus/reporter/slack/BUILD.bazel
new file mode 100644
index 0000000..19dc612
--- /dev/null
+++ b/pkg/app/lotus/reporter/slack/BUILD.bazel
@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["slack.go"],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/reporter/slack",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/app/lotus/reporter:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/app/lotus/reporter/slack/slack.go b/pkg/app/lotus/reporter/slack/slack.go
new file mode 100644
index 0000000..58cb9fd
--- /dev/null
+++ b/pkg/app/lotus/reporter/slack/slack.go
@@ -0,0 +1,101 @@
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/app/lotus/reporter"
+)
+
+type Message struct {
+	Text        string        `json:"text"`
+	UserName    string        `json:"username,omitempty"`
+	IconURL     string        `json:"icon_url,omitempty"`
+	IconEmoji   string        `json:"icon_emoji,omitempty"`
+	Attachments []*Attachment `json:"attachments,omitempty"`
+}
+
+type Attachment struct {
+	Color      string   `json:"color,omitempty"`
+	Title      string   `json:"title,omitempty"`
+	TitleLink  string   `json:"title_link,omitempty"`
+	Text       string   `json:"text,omitempty"`
+	MarkdownIn []string `json:"mrkdwn_in,omitempty"`
+}
+
+type builder struct {
+}
+
+func NewBuilder() reporter.Builder {
+	return &builder{}
+}
+
+func (b *builder) Build(r *config.Receiver, opts reporter.BuildOptions) (reporter.Reporter, error) {
+	configs, ok := r.Type.(*config.Receiver_Slack)
+	if !ok {
+		return nil, fmt.Errorf("wrong receiver type for slack: %T", r.Type)
+	}
+	return &slack{
+		hookURL: configs.Slack.HookUrl,
+		client:  http.DefaultClient,
+		logger:  opts.NamedLogger("slack-reporter"),
+	}, nil
+}
+
+type slack struct {
+	hookURL string
+	client  *http.Client
+	logger  *zap.Logger
+}
+
+func (s *slack) Report(ctx context.Context, result *model.Result) error {
+	data, err := result.Render(model.RenderFormatMarkdown)
+	if err != nil {
+		return err
+	}
+	att := &Attachment{
+		Title: fmt.Sprintf("%s %s", result.TestID, result.Status),
+		Text:  fmt.Sprintf("```%s```", string(data)),
+		Color: "danger",
+		MarkdownIn: []string{
+			"text",
+		},
+	}
+	if result.Status == model.TestSucceeded {
+		att.Color = "good"
+	}
+	msg := &Message{
+		Attachments: []*Attachment{att},
+	}
+	if err := s.send(msg); err != nil {
+		s.logger.Error("failed to report to slack", zap.Error(err))
+		return err
+	}
+	return nil
+}
+
+func (s *slack) send(msg *Message) error {
+	buf, err := json.Marshal(msg)
+	if err != nil {
+		return err
+	}
+	resp, err := s.client.Post(s.hookURL, "application/json", bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	io.Copy(ioutil.Discard, resp.Body)
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+	return nil
+}
diff --git a/pkg/app/lotus/resource/BUILD.bazel b/pkg/app/lotus/resource/BUILD.bazel
new file mode 100644
index 0000000..f933a45
--- /dev/null
+++ b/pkg/app/lotus/resource/BUILD.bazel
@@ -0,0 +1,37 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "factory.go",
+        "job.go",
+        "prometheus.go",
+        "secret.go",
+        "static_factory.go",
+        "templates.go",
+        "thanos.go",
+        "worker.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/app/lotus/resource",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/app/lotus/apis/lotus/v1beta1:go_default_library",
+        "//pkg/app/lotus/config:go_default_library",
+        "//pkg/app/lotus/model:go_default_library",
+        "//pkg/version:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+        "@io_k8s_api//apps/v1:go_default_library",
+        "@io_k8s_api//batch/v1:go_default_library",
+        "@io_k8s_api//core/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["templates_test.go"],
+    embed = [":go_default_library"],
+    deps = ["@com_github_stretchr_testify//require:go_default_library"],
+)
diff --git a/pkg/app/lotus/resource/factory.go b/pkg/app/lotus/resource/factory.go
new file mode 100644
index 0000000..9a94dfe
--- /dev/null
+++ b/pkg/app/lotus/resource/factory.go
@@ -0,0 +1,159 @@
+package resource
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+	"github.com/nghialv/lotus/pkg/app/lotus/model"
+	"github.com/nghialv/lotus/pkg/version"
+)
+
+var (
+	thanosImage     = "improbable/thanos:v0.2.0"
+	prometheusImage = "quay.io/prometheus/prometheus:v2.3.2"
+	lotusImage      = fmt.Sprintf("nghialv2607/lotus:%s", version.Get().GitCommit)
+)
+
+type ResourceFactory interface {
+	PreparerJobName() string
+	MonitorJobName() string
+	CleanerJobName() string
+	WorkerName() string
+	PrometheusName() string
+
+	NewPreparerJob() (*batchv1.Job, error)
+	NewCleanerJob() (*batchv1.Job, error)
+	NewMonitorJob() (*batchv1.Job, error)
+	NewMonitorConfigMap() (*corev1.ConfigMap, error)
+	NewWorkerDeployment() (*appsv1.Deployment, error)
+	NewWorkerService() (*corev1.Service, error)
+	NewPrometheusPod(serviceAccountName, release string) (*corev1.Pod, error)
+	NewPrometheusService() (*corev1.Service, error)
+	NewPrometheusConfigMap() (*corev1.ConfigMap, error)
+}
+
+type resourceFactory struct {
+	lotus      *lotusv1beta1.Lotus
+	configFile string
+}
+
+func NewFactory(lotus *lotusv1beta1.Lotus, configFile string) ResourceFactory {
+	return &resourceFactory{
+		lotus:      lotus,
+		configFile: configFile,
+	}
+}
+
+func (rf *resourceFactory) PreparerJobName() string {
+	return jobName(rf.lotus.Name, JobPreparer)
+}
+
+func (rf *resourceFactory) MonitorJobName() string {
+	return jobName(rf.lotus.Name, JobMonitor)
+}
+
+func (rf *resourceFactory) CleanerJobName() string {
+	return jobName(rf.lotus.Name, JobCleaner)
+}
+
+func (rf *resourceFactory) WorkerName() string {
+	return workerName(rf.lotus.Name)
+}
+
+func (rf *resourceFactory) PrometheusName() string {
+	return prometheusName(rf.lotus.Name)
+}
+
+func (rf *resourceFactory) NewPreparerJob() (*batchv1.Job, error) {
+	return newJob(
+		rf.lotus,
+		rf.lotus.Spec.Preparer.Containers,
+		rf.lotus.Spec.Preparer.Volumes,
+		JobPreparer,
+	), nil
+}
+
+func (rf *resourceFactory) NewCleanerJob() (*batchv1.Job, error) {
+	return newJob(
+		rf.lotus,
+		rf.lotus.Spec.Cleaner.Containers,
+		rf.lotus.Spec.Cleaner.Volumes,
+		JobCleaner,
+	), nil
+}
+
+func (rf *resourceFactory) NewMonitorJob() (*batchv1.Job, error) {
+	cfg, err := config.FromFile(rf.configFile)
+	if err != nil {
+		return nil, err
+	}
+	return newMonitorJob(rf.lotus, cfg), nil
+}
+
+func (rf *resourceFactory) NewMonitorConfigMap() (*corev1.ConfigMap, error) {
+	cfg, err := buildLotusConfig(rf.configFile, rf.lotus)
+	if err != nil {
+		return nil, err
+	}
+	data, err := cfg.MarshalToYaml()
+	if err != nil {
+		return nil, err
+	}
+	return newMonitorConfigMap(rf.lotus, data), nil
+}
+
+func (rf *resourceFactory) NewWorkerDeployment() (*appsv1.Deployment, error) {
+	return newWorkerDeployment(rf.lotus), nil
+}
+
+func (rf *resourceFactory) NewWorkerService() (*corev1.Service, error) {
+	return newWorkerService(rf.lotus), nil
+}
+
+func (rf *resourceFactory) NewPrometheusPod(serviceAccountName, release string) (*corev1.Pod, error) {
+	cfg, err := buildLotusConfig(rf.configFile, rf.lotus)
+	if err != nil {
+		return nil, err
+	}
+	return newPrometheusPod(rf.lotus, serviceAccountName, release, cfg)
+}
+
+func (rf *resourceFactory) NewPrometheusService() (*corev1.Service, error) {
+	return newPrometheusService(rf.lotus), nil
+}
+
+func (rf *resourceFactory) NewPrometheusConfigMap() (*corev1.ConfigMap, error) {
+	cfg, err := config.FromFile(rf.configFile)
+	if err != nil {
+		return nil, err
+	}
+	target := workerName(rf.lotus.Name)
+	return newPrometheusConfigMap(rf.lotus, target, cfg.LotusChecks())
+}
+
+func buildLotusConfig(configFile string, lotus *lotusv1beta1.Lotus) (*config.Config, error) {
+	cfg, err := config.FromFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+	cfg.DataSources = append(cfg.DataSources, clientPrometheusDataSource(lotus))
+	cfg.AddChecks(lotus.Spec.Checks...)
+	for i := range cfg.Checks {
+		if cfg.Checks[i].DataSource == "" {
+			cfg.Checks[i].DataSource = localPrometheusDataSourceName
+		}
+	}
+	return cfg, nil
+}
+
+func ownerReferences(lotus *lotusv1beta1.Lotus) []metav1.OwnerReference {
+	return []metav1.OwnerReference{
+		*metav1.NewControllerRef(lotus, model.ControllerKind),
+	}
+}
diff --git a/pkg/app/lotus/resource/job.go b/pkg/app/lotus/resource/job.go
new file mode 100644
index 0000000..97aa5e1
--- /dev/null
+++ b/pkg/app/lotus/resource/job.go
@@ -0,0 +1,148 @@
+package resource
+
+import (
+	"fmt"
+	"time"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+)
+
+type JobType string
+
+const (
+	JobPreparer JobType = "preparer"
+	JobMonitor          = "monitor"
+	JobCleaner          = "cleaner"
+)
+
+func newMonitorJob(lotus *lotusv1beta1.Lotus, cfg *config.Config) *batchv1.Job {
+	args := []string{
+		"monitor",
+		fmt.Sprintf("--test-id=%s", lotus.Name),
+		fmt.Sprintf("--run-time=%s", lotus.Spec.Worker.RunTime),
+		"--config-file=/etc/monitor/config/config.yaml",
+		fmt.Sprintf("--collect-summary-datasource=%s", localPrometheusDataSourceName),
+	}
+	if s := lotus.Spec.CheckIntervalSeconds; s != nil {
+		d := time.Duration(*s) * time.Second
+		args = append(args, fmt.Sprintf("--check-interval=%s", d.String()))
+	}
+	if s := lotus.Spec.CheckInitialDelaySeconds; s != nil {
+		d := time.Duration(*s) * time.Second
+		args = append(args, fmt.Sprintf("--check-initial-delay=%s", d.String()))
+	}
+	container := corev1.Container{
+		Name:  "monitor",
+		Image: lotusImage,
+		Args:  args,
+		Env:   []corev1.EnvVar{},
+		VolumeMounts: []corev1.VolumeMount{
+			corev1.VolumeMount{
+				Name:      "config",
+				ReadOnly:  true,
+				MountPath: "/etc/monitor/config",
+			},
+		},
+	}
+	volumes := []corev1.Volume{
+		corev1.Volume{
+			Name: "config",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: jobName(lotus.Name, JobMonitor),
+					},
+				},
+			},
+		},
+	}
+	for _, receiver := range cfg.Receivers {
+		gcsReceiver, ok := receiver.Type.(*config.Receiver_Gcs)
+		if !ok {
+			continue
+		}
+		if gcsReceiver.Gcs.Credentials != nil {
+			volumeName := fmt.Sprintf("gcs-credentials-%s", receiver.Name)
+			volumes = append(volumes, corev1.Volume{
+				Name: volumeName,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: gcsReceiver.Gcs.Credentials.Secret,
+					},
+				},
+			})
+			path := receiver.CredentialsMountPath()
+			container.VolumeMounts = append(container.VolumeMounts,
+				corev1.VolumeMount{
+					Name:      volumeName,
+					MountPath: path,
+				},
+			)
+			container.Env = append(container.Env, corev1.EnvVar{
+				Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+				Value: fmt.Sprintf("%s%s", path, gcsReceiver.Gcs.Credentials.File),
+			})
+		}
+	}
+	return newJob(
+		lotus,
+		[]corev1.Container{container},
+		volumes,
+		JobMonitor,
+	)
+}
+
+func newMonitorConfigMap(lotus *lotusv1beta1.Lotus, config []byte) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            jobName(lotus.Name, JobMonitor),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		BinaryData: map[string][]byte{
+			"config.yaml": config,
+		},
+	}
+}
+
+func newJob(lotus *lotusv1beta1.Lotus, containers []corev1.Container, volumes []corev1.Volume, jt JobType) *batchv1.Job {
+	var backoffLimit int32
+	labels := jobLabels(lotus.Name, jt)
+	return &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            jobName(lotus.Name, jt),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		Spec: batchv1.JobSpec{
+			BackoffLimit: &backoffLimit,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					RestartPolicy: corev1.RestartPolicyNever,
+					Containers:    containers,
+					Volumes:       volumes,
+				},
+			},
+		},
+	}
+}
+
+func jobName(lotusName string, jt JobType) string {
+	return fmt.Sprintf("%s-%s", lotusName, string(jt))
+}
+
+func jobLabels(lotusName string, jt JobType) map[string]string {
+	return map[string]string{
+		"app":      "lotus-job",
+		"lotus":    lotusName,
+		"job-type": string(jt),
+	}
+}
diff --git a/pkg/app/lotus/resource/prometheus.go b/pkg/app/lotus/resource/prometheus.go
new file mode 100644
index 0000000..92c0538
--- /dev/null
+++ b/pkg/app/lotus/resource/prometheus.go
@@ -0,0 +1,213 @@
+package resource
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+)
+
+const (
+	prometheusConfigDirectory     = "/etc/prometheus"
+	prometheusConfigFile          = "prometheus-config.yaml"
+	prometheusRuleFile            = "prometheus-rule.yaml"
+	prometheusPort                = 9090
+	localPrometheusDataSourceName = "_LocalPrometheus"
+	prometheusBlockDuration       = "1m"
+)
+
+func newPrometheusPod(lotus *lotusv1beta1.Lotus, serviceAccount, release string, cfg *config.Config) (*corev1.Pod, error) {
+	volumes := []corev1.Volume{
+		corev1.Volume{
+			Name: "db",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
+		corev1.Volume{
+			Name: "config",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: prometheusName(lotus.Name),
+					},
+				},
+			},
+		},
+	}
+	prometheusContainer := corev1.Container{
+		Name:  "prometheus",
+		Image: prometheusImage,
+		Args: []string{
+			fmt.Sprintf("--config.file=%s/%s", prometheusConfigDirectory, prometheusConfigFile),
+			"--storage.tsdb.path=/var/prometheus",
+			fmt.Sprintf("--storage.tsdb.min-block-duration=%s", prometheusBlockDuration),
+			fmt.Sprintf("--storage.tsdb.max-block-duration=%s", prometheusBlockDuration),
+			"--storage.tsdb.retention=6h",
+			"--web.enable-lifecycle",
+		},
+		Ports: []corev1.ContainerPort{
+			corev1.ContainerPort{
+				Name:          "prom-http",
+				ContainerPort: prometheusPort,
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			corev1.VolumeMount{
+				Name:      "config",
+				MountPath: prometheusConfigDirectory,
+			},
+			corev1.VolumeMount{
+				Name:      "db",
+				MountPath: "/var/prometheus",
+			},
+		},
+	}
+	thanosContainer := corev1.Container{
+		Name:  "thanos-sidecar",
+		Image: thanosImage,
+		Args: []string{
+			"sidecar",
+			"--tsdb.path=/var/prometheus",
+			fmt.Sprintf("--prometheus.url=http://127.0.0.1:%d", prometheusPort),
+			"--cluster.disable",
+		},
+		Env:   []corev1.EnvVar{},
+		Ports: thanosPorts(),
+		VolumeMounts: []corev1.VolumeMount{
+			corev1.VolumeMount{
+				Name:      "config",
+				MountPath: prometheusConfigDirectory,
+			},
+			corev1.VolumeMount{
+				Name:      "db",
+				MountPath: "/var/prometheus",
+			},
+		},
+	}
+	if cfg.TimeSeriesStorage != nil {
+		setTimeSeriesStoreConfig(&thanosContainer, &volumes, release)
+		if gcs, ok := cfg.TimeSeriesStorage.Type.(*config.TimeSeriesStorage_Gcs); ok {
+			if gcs.Gcs.Credentials != nil {
+				setGCSCredentials(&thanosContainer, &volumes, gcs.Gcs.Credentials)
+			}
+		}
+	}
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            prometheusName(lotus.Name),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+			Labels:          prometheusPodLabels(lotus.Name, release),
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy: corev1.RestartPolicyNever,
+			Containers: []corev1.Container{
+				prometheusContainer,
+				thanosContainer,
+			},
+			Volumes: volumes,
+		},
+	}
+	if serviceAccount != "" {
+		pod.Spec.ServiceAccountName = serviceAccount
+	}
+	return pod, nil
+}
+
+func newPrometheusService(lotus *lotusv1beta1.Lotus) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            prometheusName(lotus.Name),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: prometheusServiceLabels(lotus.Name),
+			Ports: []corev1.ServicePort{
+				corev1.ServicePort{
+					Name:       "metrics",
+					TargetPort: intstr.FromInt(prometheusPort),
+					Port:       int32(prometheusPort),
+				},
+			},
+		},
+	}
+}
+
+func newPrometheusConfigMap(lotus *lotusv1beta1.Lotus, target string, globalChecks []lotusv1beta1.LotusCheck) (*corev1.ConfigMap, error) {
+	config, err := renderTemplate(
+		&prometheusConfigParams{
+			Name:        prometheusName(lotus.Name),
+			Namespace:   lotus.Namespace,
+			ServiceName: target,
+			RuleFiles: []string{
+				fmt.Sprintf("%s/%s", prometheusConfigDirectory, prometheusRuleFile),
+			},
+		},
+		prometheusConfigTemplate,
+	)
+	if err != nil {
+		return nil, err
+	}
+	globalChecks = append(globalChecks, lotus.Spec.Checks...)
+	rule, err := renderTemplate(
+		&prometheusRuleParams{
+			Alerts: globalChecks,
+		},
+		prometheusRuleTemplate,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            prometheusName(lotus.Name),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		BinaryData: map[string][]byte{
+			prometheusConfigFile: config,
+			prometheusRuleFile:   rule,
+		},
+	}, nil
+}
+
+func prometheusName(lotusName string) string {
+	return fmt.Sprintf("%s-prometheus", lotusName)
+}
+
+func prometheusServiceLabels(lotusName string) map[string]string {
+	return map[string]string{
+		"app":   "lotus-prometheus",
+		"lotus": lotusName,
+	}
+}
+
+func prometheusPodLabels(lotusName, release string) map[string]string {
+	return map[string]string{
+		"app":           "lotus-prometheus",
+		"lotus":         lotusName,
+		thanosPeerLabel: release,
+	}
+}
+
+func clientPrometheusDataSource(lotus *lotusv1beta1.Lotus) *config.DataSource {
+	address := fmt.Sprintf("http://%s.%s.svc.cluster.local:%d",
+		prometheusName(lotus.Name),
+		lotus.Namespace,
+		prometheusPort,
+	)
+	return &config.DataSource{
+		Name: localPrometheusDataSourceName,
+		Type: &config.DataSource_Prometheus{
+			Prometheus: &config.PrometheusConfigs{
+				Address: address,
+			},
+		},
+	}
+}
diff --git a/pkg/app/lotus/resource/secret.go b/pkg/app/lotus/resource/secret.go
new file mode 100644
index 0000000..5161837
--- /dev/null
+++ b/pkg/app/lotus/resource/secret.go
@@ -0,0 +1,59 @@
+package resource
+
+import (
+	"fmt"
+
+	"github.com/ghodss/yaml"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+)
+
+const (
+	timeSeriesStoreConfigFile = "store-config.yaml"
+)
+
+type ThanosStore struct {
+	Type   string      `json:"type"`
+	Config interface{} `json:"config"`
+}
+
+type ThanosGCSConfig struct {
+	Bucket string `json:"bucket"`
+}
+
+func generateThanosStoreConfig(cfg *config.TimeSeriesStorage) ([]byte, error) {
+	switch store := cfg.Type.(type) {
+	case *config.TimeSeriesStorage_Gcs:
+		return yaml.Marshal(&ThanosStore{
+			Type: "GCS",
+			Config: &ThanosGCSConfig{
+				Bucket: store.Gcs.Bucket,
+			},
+		})
+	default:
+		return nil, fmt.Errorf("unsupported store: %v", cfg.Type)
+	}
+}
+
+func newTimeSeriesStoreConfigSecret(namespace, release string, cfg *config.TimeSeriesStorage, owners []metav1.OwnerReference) (*corev1.Secret, error) {
+	data, err := generateThanosStoreConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            timeSeriesStoreConfigSecretName(release),
+			Namespace:       namespace,
+			OwnerReferences: owners,
+		},
+		Data: map[string][]byte{
+			timeSeriesStoreConfigFile: data,
+		},
+	}, nil
+}
+
+func timeSeriesStoreConfigSecretName(release string) string {
+	return fmt.Sprintf("%s-time-series-store-config", release)
+}
diff --git a/pkg/app/lotus/resource/static_factory.go b/pkg/app/lotus/resource/static_factory.go
new file mode 100644
index 0000000..2e0be17
--- /dev/null
+++ b/pkg/app/lotus/resource/static_factory.go
@@ -0,0 +1,82 @@
+package resource
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+)
+
+type StaticResourceFactory interface {
+	ThanosStoreName() string
+	ThanosQueryName() string
+	ThanosPeerName() string
+	TimeSeriesStoreConfigSecretName() string
+
+	NewThanosStoreStatefulSet() (*appsv1.StatefulSet, error)
+	NewThanosQueryDeployment() (*appsv1.Deployment, error)
+	NewThanosQueryService() (*corev1.Service, error)
+	NewThanosPeerService() (*corev1.Service, error)
+	NewTimeSeriesStoreConfigSecret() (*corev1.Secret, error)
+}
+
+type staticResourceFactory struct {
+	namespace       string
+	release         string
+	configFile      string
+	ownerReferences []metav1.OwnerReference
+}
+
+func NewStaticResourceFactory(namespace, release, configFile string, owners []metav1.OwnerReference) StaticResourceFactory {
+	return &staticResourceFactory{
+		namespace:       namespace,
+		release:         release,
+		configFile:      configFile,
+		ownerReferences: owners,
+	}
+}
+
+func (f *staticResourceFactory) ThanosStoreName() string {
+	return thanosStoreName(f.release)
+}
+
+func (f *staticResourceFactory) ThanosQueryName() string {
+	return thanosQueryName(f.release)
+}
+
+func (f *staticResourceFactory) ThanosPeerName() string {
+	return thanosPeerName(f.release)
+}
+
+func (f *staticResourceFactory) TimeSeriesStoreConfigSecretName() string {
+	return timeSeriesStoreConfigSecretName(f.release)
+}
+
+func (f *staticResourceFactory) NewThanosStoreStatefulSet() (*appsv1.StatefulSet, error) {
+	cfg, err := config.FromFile(f.configFile)
+	if err != nil {
+		return nil, err
+	}
+	return newThanosStoreStatefulSet(f.namespace, f.release, cfg.TimeSeriesStorage, f.ownerReferences)
+}
+
+func (f *staticResourceFactory) NewThanosQueryDeployment() (*appsv1.Deployment, error) {
+	return newThanosQueryDeployment(f.namespace, f.release, f.ownerReferences), nil
+}
+
+func (f *staticResourceFactory) NewThanosQueryService() (*corev1.Service, error) {
+	return newThanosQueryService(f.namespace, f.release, f.ownerReferences), nil
+}
+
+func (f *staticResourceFactory) NewThanosPeerService() (*corev1.Service, error) {
+	return newThanosPeerService(f.namespace, f.release, f.ownerReferences), nil
+}
+
+func (f *staticResourceFactory) NewTimeSeriesStoreConfigSecret() (*corev1.Secret, error) {
+	cfg, err := config.FromFile(f.configFile)
+	if err != nil {
+		return nil, err
+	}
+	return newTimeSeriesStoreConfigSecret(f.namespace, f.release, cfg.TimeSeriesStorage, f.ownerReferences)
+}
diff --git a/pkg/app/lotus/resource/templates.go b/pkg/app/lotus/resource/templates.go
new file mode 100644
index 0000000..5645c2f
--- /dev/null
+++ b/pkg/app/lotus/resource/templates.go
@@ -0,0 +1,123 @@
+package resource
+
+import (
+	"bytes"
+	"text/template"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+)
+
+func renderTemplate(params interface{}, tpl string) ([]byte, error) {
+	template, err := template.New("template").Parse(tpl)
+	if err != nil {
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = template.Execute(&buffer, params)
+	if err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+type prometheusConfigParams struct {
+	Name        string
+	Namespace   string
+	ServiceName string
+	RuleFiles   []string
+}
+
+const prometheusConfigTemplate = `
+global:
+  scrape_interval: 5s
+  scrape_timeout: 5s
+  evaluation_interval: 5s
+  external_labels:
+    monitor: prometheus
+    replica: {{ .Name }}
+scrape_configs:
+- job_name: lotus-runer
+  metrics_path: /metrics
+  scheme: http
+  kubernetes_sd_configs:
+  - api_server: null
+    role: endpoints
+    namespaces:
+      names:
+      - {{ .Namespace  }}
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_name]
+    separator: ;
+    regex: {{ .ServiceName }}
+    replacement: $1
+    action: keep
+  - source_labels: [__meta_kubernetes_namespace]
+    separator: ;
+    regex: (.*)
+    target_label: namespace
+    replacement: $1
+    action: replace
+  - source_labels: [__meta_kubernetes_pod_name]
+    separator: ;
+    regex: (.*)
+    target_label: pod
+    replacement: $1
+    action: replace
+  - source_labels: [__meta_kubernetes_service_name]
+    separator: ;
+    regex: (.*)
+    target_label: service
+    replacement: $1
+    action: replace
+  - source_labels: [__meta_kubernetes_service_name]
+    separator: ;
+    regex: (.*)
+    target_label: job
+    replacement: ${1}
+    action: replace
+{{- if gt (len .RuleFiles) 0 }}
+rule_files:
+{{- range .RuleFiles }}
+  - {{ . }}
+{{- end }}
+{{- end }}
+`
+
+type prometheusRuleParams struct {
+	Alerts []v1beta1.LotusCheck
+}
+
+const prometheusRuleTemplate = `
+groups:
+- name: lotus
+  rules:
+{{- range .Alerts }}
+  - alert: {{ .Name }}
+    expr: {{ .Expr }}
+    for: {{ .For }}
+{{- end }}
+  - record: lotus_virtual_user_failure_percentage
+    expr: 100 * sum by (job) (lotus_virtual_user_count{virtual_user_status="failed"}) / sum by (job) (lotus_virtual_user_count{virtual_user_status="started"})
+  - record: lotus_grpc_client_completed_rpcs_per_second:method
+    expr: sum by (job, grpc_client_method) (rate(lotus_grpc_client_completed_rpcs[1m]))
+  - record: lotus_grpc_client_completed_rpcs_per_second:status
+    expr: sum by (job, grpc_client_status) (rate(lotus_grpc_client_completed_rpcs[1m]))
+  - record: lotus_grpc_client_completed_rpcs_failure_percentage:method
+    expr: 100 * sum by (job, grpc_client_method) (rate(lotus_grpc_client_completed_rpcs{grpc_client_status!~"OK|NOT_FOUND|ALREADY_EXISTS"}[1m])) / sum by (job, grpc_client_method) (rate(lotus_grpc_client_completed_rpcs[1m]))
+  - record: lotus_grpc_client_roundtrip_latency:method
+    expr: sum by (job, grpc_client_method) (rate(lotus_grpc_client_roundtrip_latency_sum[1m])) / sum by (job, grpc_client_method) (rate(lotus_grpc_client_roundtrip_latency_count[1m]))
+  - record: lotus_grpc_client_sent_bytes_per_rpc:method
+    expr: sum by (job, grpc_client_method) (rate(lotus_grpc_client_sent_bytes_per_rpc_sum[1m])) / sum by (job, grpc_client_method) (rate(lotus_grpc_client_sent_bytes_per_rpc_count[1m]))
+  - record: lotus_grpc_client_received_bytes_per_rpc:method
+    expr: sum by (job, grpc_client_method) (rate(lotus_grpc_client_received_bytes_per_rpc_sum[1m])) / sum by (job, grpc_client_method) (rate(lotus_grpc_client_received_bytes_per_rpc_count[1m]))
+  - record: lotus_http_client_completed_requests_per_second:host:route:method
+    expr: sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_completed_count[1m]))
+  - record: lotus_http_client_completed_requests_5xx_percentage:host:route:method
+    expr: 100 * sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_completed_count{http_client_status=~"5.."}[1m])) / sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_completed_count[1m]))
+  - record: lotus_http_client_roundtrip_latency:host:route:method
+    expr: sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_roundtrip_latency_sum[1m])) / sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_roundtrip_latency_count[1m]))
+  - record: lotus_http_client_sent_bytes:host:route:method
+    expr: sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_sent_bytes_sum[1m])) / sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_sent_bytes_count[1m]))
+  - record: lotus_http_client_received_bytes:host:route:method
+    expr: sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_received_bytes_sum[1m])) / sum by (job, http_client_host, http_client_route, http_client_method) (rate(lotus_http_client_received_bytes_count[1m]))
+`
diff --git a/pkg/app/lotus/resource/templates_test.go b/pkg/app/lotus/resource/templates_test.go
new file mode 100644
index 0000000..13f87dd
--- /dev/null
+++ b/pkg/app/lotus/resource/templates_test.go
@@ -0,0 +1,22 @@
+package resource
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRenderTemplate(t *testing.T) {
+	params := &prometheusConfigParams{
+		Namespace:   "default",
+		ServiceName: "foo",
+		RuleFiles: []string{
+			"rule-file-1.yaml",
+			"rule-file-2.yaml",
+		},
+	}
+	cfg, err := renderTemplate(params, prometheusConfigTemplate)
+	require.NoError(t, err)
+	fmt.Println(string(cfg))
+}
diff --git a/pkg/app/lotus/resource/thanos.go b/pkg/app/lotus/resource/thanos.go
new file mode 100644
index 0000000..36d93ea
--- /dev/null
+++ b/pkg/app/lotus/resource/thanos.go
@@ -0,0 +1,242 @@
+package resource
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"github.com/nghialv/lotus/pkg/app/lotus/config"
+)
+
+const (
+	thanosPeerLabel = "lotus-thanos-peer"
+)
+
+func newThanosStoreStatefulSet(namespace, release string, cfg *config.TimeSeriesStorage, owners []metav1.OwnerReference) (*appsv1.StatefulSet, error) {
+	volumes := []corev1.Volume{
+		corev1.Volume{
+			Name: "data",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
+	}
+	container := corev1.Container{
+		Name:  "thanos-store",
+		Image: thanosImage,
+		Args: []string{
+			"store",
+			"--data-dir=/var/thanos/store",
+			"--cluster.disable",
+		},
+		Env:   []corev1.EnvVar{},
+		Ports: thanosPorts(),
+		VolumeMounts: []corev1.VolumeMount{
+			corev1.VolumeMount{
+				Name:      "data",
+				MountPath: "/var/thanos/store",
+			},
+		},
+	}
+	if cfg != nil {
+		setTimeSeriesStoreConfig(&container, &volumes, release)
+		if gcs, ok := cfg.Type.(*config.TimeSeriesStorage_Gcs); ok {
+			if gcs.Gcs.Credentials != nil {
+				setGCSCredentials(&container, &volumes, gcs.Gcs.Credentials)
+			}
+		}
+	}
+
+	labels := thanosStoreLabels(release)
+	replicas := int32(1)
+
+	statefulset := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            thanosStoreName(release),
+			Namespace:       namespace,
+			OwnerReferences: owners,
+			Labels:          labels,
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{container},
+					Volumes:    volumes,
+				},
+			},
+		},
+	}
+	return statefulset, nil
+}
+
+func newThanosPeerService(namespace, release string, owners []metav1.OwnerReference) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            thanosPeerName(release),
+			Namespace:       namespace,
+			OwnerReferences: owners,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector:  thanosPeerLabels(release),
+			ClusterIP: "None",
+			Ports: []corev1.ServicePort{
+				corev1.ServicePort{
+					Name:       "grpc",
+					TargetPort: intstr.FromString("grpc"),
+					Port:       int32(10901),
+				},
+			},
+		},
+	}
+}
+
+func newThanosQueryDeployment(namespace, release string, owners []metav1.OwnerReference) *appsv1.Deployment {
+	replicas := int32(1)
+	labels := thanosQueryLabels(release)
+	containers := []corev1.Container{
+		corev1.Container{
+			Name:  "thanos-query",
+			Image: thanosImage,
+			Args: []string{
+				"query",
+				"--query.replica-label=replica",
+				"--cluster.disable",
+				fmt.Sprintf("--store=dns+%s.%s.svc.cluster.local:10901", thanosPeerName(release), namespace),
+			},
+			Ports: thanosPorts(),
+		},
+	}
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            thanosQueryName(release),
+			Namespace:       namespace,
+			OwnerReferences: owners,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					RestartPolicy: corev1.RestartPolicyAlways,
+					Containers:    containers,
+				},
+			},
+		},
+	}
+}
+
+func newThanosQueryService(namespace, release string, owners []metav1.OwnerReference) *corev1.Service {
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            thanosQueryName(release),
+			Namespace:       namespace,
+			OwnerReferences: owners,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: thanosQueryLabels(release),
+			Ports: []corev1.ServicePort{
+				corev1.ServicePort{
+					Name:       "http",
+					TargetPort: intstr.FromString("http"),
+					Port:       int32(9090),
+				},
+			},
+		},
+	}
+}
+
+func thanosStoreName(release string) string {
+	return fmt.Sprintf("%s-thanos-store", release)
+}
+
+func thanosPeerName(release string) string {
+	return fmt.Sprintf("%s-thanos-peers", release)
+}
+
+func thanosQueryName(release string) string {
+	return fmt.Sprintf("%s-thanos-query", release)
+}
+
+func thanosStoreLabels(release string) map[string]string {
+	return map[string]string{
+		"app":           thanosStoreName(release),
+		thanosPeerLabel: release,
+	}
+}
+
+func thanosPeerLabels(release string) map[string]string {
+	return map[string]string{
+		thanosPeerLabel: release,
+	}
+}
+
+func thanosQueryLabels(release string) map[string]string {
+	return map[string]string{
+		"app": thanosQueryName(release),
+	}
+}
+
+func thanosPorts() []corev1.ContainerPort {
+	return []corev1.ContainerPort{
+		corev1.ContainerPort{
+			Name:          "http",
+			ContainerPort: 10902,
+		},
+		corev1.ContainerPort{
+			Name:          "grpc",
+			ContainerPort: 10901,
+		},
+	}
+}
+
+func setGCSCredentials(container *corev1.Container, volumes *[]corev1.Volume, credentials *config.SecretFileSelector) {
+	container.Env = append(container.Env, corev1.EnvVar{
+		Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+		Value: fmt.Sprintf("/creds/gcs/%s", credentials.File),
+	})
+	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+		Name:      "gcs-credentials",
+		MountPath: "/creds/gcs/",
+	})
+	*volumes = append(*volumes, corev1.Volume{
+		Name: "gcs-credentials",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: credentials.Secret,
+			},
+		},
+	})
+}
+
+func setTimeSeriesStoreConfig(container *corev1.Container, volumes *[]corev1.Volume, release string) {
+	container.Args = append(container.Args,
+		fmt.Sprintf("--objstore.config-file=/creds/objstore/%s", timeSeriesStoreConfigFile),
+	)
+	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+		Name:      "time-series-store-config",
+		MountPath: "/creds/objstore/",
+	})
+	*volumes = append(*volumes, corev1.Volume{
+		Name: "time-series-store-config",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: timeSeriesStoreConfigSecretName(release),
+			},
+		},
+	})
+}
diff --git a/pkg/app/lotus/resource/worker.go b/pkg/app/lotus/resource/worker.go
new file mode 100644
index 0000000..145d99b
--- /dev/null
+++ b/pkg/app/lotus/resource/worker.go
@@ -0,0 +1,72 @@
+package resource
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	lotusv1beta1 "github.com/nghialv/lotus/pkg/app/lotus/apis/lotus/v1beta1"
+)
+
+func newWorkerDeployment(lotus *lotusv1beta1.Lotus) *appsv1.Deployment {
+	labels := workerLabels(lotus.Name)
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            workerName(lotus.Name),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: lotus.Spec.Worker.Replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					RestartPolicy: corev1.RestartPolicyAlways,
+					Containers:    lotus.Spec.Worker.Containers,
+				},
+			},
+		},
+	}
+}
+
+func newWorkerService(lotus *lotusv1beta1.Lotus) *corev1.Service {
+	labels := workerLabels(lotus.Name)
+	metricsPort := *lotus.Spec.Worker.MetricsPort
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            workerName(lotus.Name),
+			Namespace:       lotus.Namespace,
+			OwnerReferences: ownerReferences(lotus),
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: labels,
+			Ports: []corev1.ServicePort{
+				corev1.ServicePort{
+					Name:       "metrics",
+					TargetPort: intstr.FromInt(int(metricsPort)),
+					Port:       metricsPort,
+				},
+			},
+		},
+	}
+
+}
+
+func workerName(lotusName string) string {
+	return fmt.Sprintf("%s-worker", lotusName)
+}
+
+func workerLabels(lotusName string) map[string]string {
+	return map[string]string{
+		"app":   "lotus-worker",
+		"lotus": lotusName,
+	}
+}
diff --git a/pkg/cli/BUILD.bazel b/pkg/cli/BUILD.bazel
new file mode 100644
index 0000000..b2ea292
--- /dev/null
+++ b/pkg/cli/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "app.go",
+        "cmd.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/cli",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/log:go_default_library",
+        "//pkg/version:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+        "@org_uber_go_zap//:go_default_library",
+    ],
+)
diff --git a/pkg/cli/app.go b/pkg/cli/app.go
new file mode 100644
index 0000000..713ecff
--- /dev/null
+++ b/pkg/cli/app.go
@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/nghialv/lotus/pkg/log"
+	"github.com/nghialv/lotus/pkg/version"
+)
+
+type App struct {
+	rootCmd     *cobra.Command
+	logLevel    string
+	logEncoding string
+}
+
+func NewApp(name, desc string) *App {
+	a := &App{
+		rootCmd: &cobra.Command{
+			Use:   name,
+			Short: desc,
+		},
+		logLevel:    log.DefaultLevel,
+		logEncoding: log.DefaultEncoding,
+	}
+	versionCmd := &cobra.Command{
+		Use:   "version",
+		Short: "Print the information of current binary",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Println(version.Get())
+		},
+	}
+	a.rootCmd.AddCommand(versionCmd)
+	a.setGlobalFlags()
+	return a
+}
+
+func (a *App) AddCommands(cmds ...*cobra.Command) {
+	for _, cmd := range cmds {
+		a.rootCmd.AddCommand(cmd)
+	}
+}
+
+func (a *App) Run() error {
+	return a.rootCmd.Execute()
+}
+
+func (a *App) setGlobalFlags() {
+	a.rootCmd.PersistentFlags().StringVar(&a.logLevel, "log-level", a.logLevel, "The minimum enabled logging level")
+	a.rootCmd.PersistentFlags().StringVar(&a.logEncoding, "log-encoding", a.logEncoding, "The encoding type for logger [json|console]")
+}
diff --git a/pkg/cli/cmd.go b/pkg/cli/cmd.go
new file mode 100644
index 0000000..955e211
--- /dev/null
+++ b/pkg/cli/cmd.go
@@ -0,0 +1,64 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/nghialv/lotus/pkg/log"
+	"github.com/nghialv/lotus/pkg/version"
+)
+
+func WithContext(runner func(context.Context, *zap.Logger) error) func(cmd *cobra.Command, args []string) error {
+	return func(cmd *cobra.Command, args []string) error {
+		return runWithContext(cmd, runner)
+	}
+}
+
+func runWithContext(cmd *cobra.Command, runner func(context.Context, *zap.Logger) error) error {
+	logger, err := newLogger(cmd)
+	if err != nil {
+		return err
+	}
+	defer logger.Sync()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+	defer signal.Stop(ch)
+
+	go func() {
+		select {
+		case s := <-ch:
+			logger.Info("stopping due to signal", zap.Stringer("signal", s))
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+
+	logger.Info(fmt.Sprintf("running %s", cmd.CommandPath()))
+	return runner(ctx, logger)
+}
+
+func newLogger(cmd *cobra.Command) (*zap.Logger, error) {
+	configs := log.DefaultConfigs
+	configs.ServiceContext = &log.ServiceContext{
+		Service: strings.Replace(cmd.CommandPath(), " ", ".", -1),
+		Version: version.Get().Version,
+	}
+	if f := cmd.Flag("log-level"); f != nil {
+		configs.Level = f.Value.String()
+	}
+	if f := cmd.Flag("log-encoding"); f != nil {
+		configs.Encoding = f.Value.String()
+	}
+	return log.NewLogger(configs)
+}
diff --git a/pkg/log/BUILD.bazel b/pkg/log/BUILD.bazel
new file mode 100644
index 0000000..0798b58
--- /dev/null
+++ b/pkg/log/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",  #keep
+    srcs = ["log_test.go"],
+    embed = [":go_default_library"],
+    deps = ["@com_github_stretchr_testify//assert:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["log.go"],
+    importpath = "github.com/nghialv/lotus/pkg/log",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@org_uber_go_zap//:go_default_library",
+        "@org_uber_go_zap//zapcore:go_default_library",
+    ],
+)
diff --git a/pkg/log/log.go b/pkg/log/log.go
new file mode 100644
index 0000000..ff79450
--- /dev/null
+++ b/pkg/log/log.go
@@ -0,0 +1,108 @@
+package log
+
+import (
+	"errors"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+const (
+	DefaultLevel    = "info"
+	DefaultEncoding = "json"
+)
+
+var (
+	DefaultConfigs = Configs{
+		Level:    DefaultLevel,
+		Encoding: DefaultEncoding,
+	}
+)
+
+type Configs struct {
+	Level          string
+	Encoding       string
+	ServiceContext *ServiceContext
+}
+
+func NewLogger(c Configs) (*zap.Logger, error) {
+	level := new(zapcore.Level)
+	if err := level.Set(c.Level); err != nil {
+		return nil, err
+	}
+	var options []zap.Option
+	if c.ServiceContext != nil {
+		options = []zap.Option{
+			zap.Fields(zap.Object("serviceContext", c.ServiceContext)),
+		}
+	}
+	logger, err := newConfig(*level, c.Encoding).Build(options...)
+	if err != nil {
+		return nil, err
+	}
+	return logger.Named(c.ServiceContext.Service), nil
+}
+
+func newConfig(level zapcore.Level, encoding string) zap.Config {
+	return zap.Config{
+		Level:       zap.NewAtomicLevelAt(level),
+		Development: false,
+		Sampling: &zap.SamplingConfig{
+			Initial:    100,
+			Thereafter: 100,
+		},
+		Encoding:         encoding,
+		EncoderConfig:    newEncoderConfig(),
+		OutputPaths:      []string{"stderr"},
+		ErrorOutputPaths: []string{"stderr"},
+	}
+}
+
+func newEncoderConfig() zapcore.EncoderConfig {
+	return zapcore.EncoderConfig{
+		TimeKey:        "eventTime",
+		LevelKey:       "severity",
+		NameKey:        "logger",
+		CallerKey:      "caller",
+		MessageKey:     "message",
+		StacktraceKey:  "stacktrace",
+		LineEnding:     zapcore.DefaultLineEnding,
+		EncodeLevel:    encodeLevel,
+		EncodeTime:     zapcore.EpochTimeEncoder,
+		EncodeDuration: zapcore.SecondsDurationEncoder,
+		EncodeCaller:   zapcore.ShortCallerEncoder,
+	}
+}
+
+func encodeLevel(l zapcore.Level, enc zapcore.PrimitiveArrayEncoder) {
+	switch l {
+	case zapcore.DebugLevel:
+		enc.AppendString("DEBUG")
+	case zapcore.InfoLevel:
+		enc.AppendString("INFO")
+	case zapcore.WarnLevel:
+		enc.AppendString("WARNING")
+	case zapcore.ErrorLevel:
+		enc.AppendString("ERROR")
+	case zapcore.DPanicLevel:
+		enc.AppendString("CRITICAL")
+	case zapcore.PanicLevel:
+		enc.AppendString("ALERT")
+	case zapcore.FatalLevel:
+		enc.AppendString("EMERGENCY")
+	}
+}
+
+type ServiceContext struct {
+	Service string
+	Version string
+}
+
+func (sc ServiceContext) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	if sc.Service == "" {
+		return errors.New("service name is mandatory")
+	}
+	enc.AddString("service", sc.Service)
+	enc.AddString("version", sc.Version)
+	return nil
+}
diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go
new file mode 100644
index 0000000..fe857e5
--- /dev/null
+++ b/pkg/log/log_test.go
@@ -0,0 +1,59 @@
+package log
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewLoggerOK(t *testing.T) {
+	validLevels := []string{
+		"debug",
+		"info",
+		"warn",
+		"error",
+		"dpanic",
+		"panic",
+		"fatal",
+	}
+	validEncodings := []string{
+		"json",
+		"console",
+	}
+	for _, level := range validLevels {
+		for _, encoding := range validEncodings {
+			cfg := Configs{
+				Level:    level,
+				Encoding: encoding,
+				ServiceContext: &ServiceContext{
+					Service: "test-service",
+					Version: "1.0.0",
+				},
+			}
+			logger, err := NewLogger(cfg)
+			des := fmt.Sprintf("level: %s, encoding: %s", level, encoding)
+			assert.Nil(t, err, des)
+			assert.NotNil(t, logger, des)
+		}
+	}
+}
+
+func TestNewLoggerFailed(t *testing.T) {
+	configs := []Configs{
+		Configs{
+			Level:    "foo",
+			Encoding: "json",
+		},
+		Configs{
+			Level:    "info",
+			Encoding: "foo",
+		},
+	}
+	for _, cfg := range configs {
+		logger, err := NewLogger(cfg)
+		des := fmt.Sprintf("level: %s, encoding: %s", cfg.Level, cfg.Encoding)
+		assert.NotNil(t, err, des)
+		assert.Nil(t, logger, des)
+	}
+}
diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel
new file mode 100644
index 0000000..478f767
--- /dev/null
+++ b/pkg/metrics/BUILD.bazel
@@ -0,0 +1,25 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "logger.go",
+        "metrics.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/metrics",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/metrics/grpcmetrics:go_default_library",
+        "//pkg/metrics/httpmetrics:go_default_library",
+        "//pkg/virtualuser:go_default_library",
+        "@io_opencensus_go//exporter/prometheus:go_default_library",
+        "@io_opencensus_go//stats/view:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["metrics_test.go"],
+    embed = [":go_default_library"],
+)
diff --git a/pkg/metrics/grpcmetrics/BUILD.bazel b/pkg/metrics/grpcmetrics/BUILD.bazel
new file mode 100644
index 0000000..266b76e
--- /dev/null
+++ b/pkg/metrics/grpcmetrics/BUILD.bazel
@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "common.go",
+        "grpc.go",
+        "grpc_stats.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/metrics/grpcmetrics",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@io_opencensus_go//stats:go_default_library",
+        "@io_opencensus_go//stats/view:go_default_library",
+        "@io_opencensus_go//tag:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+        "@org_golang_google_grpc//grpclog:go_default_library",
+        "@org_golang_google_grpc//stats:go_default_library",
+        "@org_golang_google_grpc//status:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
+    ],
+)
diff --git a/pkg/metrics/grpcmetrics/common.go b/pkg/metrics/grpcmetrics/common.go
new file mode 100644
index 0000000..b3da913
--- /dev/null
+++ b/pkg/metrics/grpcmetrics/common.go
@@ -0,0 +1,163 @@
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcmetrics
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	ocstats "go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+type rpcData struct {
+	sentCount, sentBytes, recvCount, recvBytes int64
+	startTime                                  time.Time
+	method                                     string
+}
+
+type grpcInstrumentationKey string
+
+var (
+	rpcDataKey = grpcInstrumentationKey("lotus-rpcData")
+)
+
+func methodName(fullname string) string {
+	return strings.TrimLeft(fullname, "/")
+}
+
+func statsHandleRPC(ctx context.Context, s stats.RPCStats) {
+	switch st := s.(type) {
+	case *stats.Begin, *stats.OutHeader, *stats.InHeader, *stats.InTrailer, *stats.OutTrailer:
+		// do nothing for client
+	case *stats.OutPayload:
+		handleRPCOutPayload(ctx, st)
+	case *stats.InPayload:
+		handleRPCInPayload(ctx, st)
+	case *stats.End:
+		handleRPCEnd(ctx, st)
+	default:
+		grpclog.Infof("unexpected stats: %T", st)
+	}
+}
+
+func handleRPCOutPayload(ctx context.Context, s *stats.OutPayload) {
+	d, ok := ctx.Value(rpcDataKey).(*rpcData)
+	if !ok {
+		if grpclog.V(2) {
+			grpclog.Infoln("Failed to retrieve *rpcData from context.")
+		}
+		return
+	}
+	atomic.AddInt64(&d.sentBytes, int64(s.Length))
+	atomic.AddInt64(&d.sentCount, 1)
+}
+
+func handleRPCInPayload(ctx context.Context, s *stats.InPayload) {
+	d, ok := ctx.Value(rpcDataKey).(*rpcData)
+	if !ok {
+		if grpclog.V(2) {
+			grpclog.Infoln("Failed to retrieve *rpcData from context.")
+		}
+		return
+	}
+	atomic.AddInt64(&d.recvBytes, int64(s.Length))
+	atomic.AddInt64(&d.recvCount, 1)
+}
+
+func handleRPCEnd(ctx context.Context, s *stats.End) {
+	d, ok := ctx.Value(rpcDataKey).(*rpcData)
+	if !ok {
+		if grpclog.V(2) {
+			grpclog.Infoln("Failed to retrieve *rpcData from context.")
+		}
+		return
+	}
+	elapsedTime := time.Since(d.startTime)
+
+	var st string
+	if s.Error != nil {
+		s, ok := status.FromError(s.Error)
+		if ok {
+			st = statusCodeToString(s)
+		}
+	} else {
+		st = "OK"
+	}
+
+	latencyMillis := float64(elapsedTime) / float64(time.Millisecond)
+	if !s.Client {
+		return
+	}
+	ocstats.RecordWithTags(ctx,
+		[]tag.Mutator{
+			tag.Upsert(KeyClientMethod, methodName(d.method)),
+			tag.Upsert(KeyClientStatus, st),
+		},
+		ClientSentBytesPerRPC.M(atomic.LoadInt64(&d.sentBytes)),
+		ClientSentMessagesPerRPC.M(atomic.LoadInt64(&d.sentCount)),
+		ClientReceivedMessagesPerRPC.M(atomic.LoadInt64(&d.recvCount)),
+		ClientReceivedBytesPerRPC.M(atomic.LoadInt64(&d.recvBytes)),
+		ClientRoundtripLatency.M(latencyMillis))
+}
+
+func statusCodeToString(s *status.Status) string {
+	// see https://github.com/grpc/grpc/blob/master/doc/statuscodes.md
+	switch c := s.Code(); c {
+	case codes.OK:
+		return "OK"
+	case codes.Canceled:
+		return "CANCELLED"
+	case codes.Unknown:
+		return "UNKNOWN"
+	case codes.InvalidArgument:
+		return "INVALID_ARGUMENT"
+	case codes.DeadlineExceeded:
+		return "DEADLINE_EXCEEDED"
+	case codes.NotFound:
+		return "NOT_FOUND"
+	case codes.AlreadyExists:
+		return "ALREADY_EXISTS"
+	case codes.PermissionDenied:
+		return "PERMISSION_DENIED"
+	case codes.ResourceExhausted:
+		return "RESOURCE_EXHAUSTED"
+	case codes.FailedPrecondition:
+		return "FAILED_PRECONDITION"
+	case codes.Aborted:
+		return "ABORTED"
+	case codes.OutOfRange:
+		return "OUT_OF_RANGE"
+	case codes.Unimplemented:
+		return "UNIMPLEMENTED"
+	case codes.Internal:
+		return "INTERNAL"
+	case codes.Unavailable:
+		return "UNAVAILABLE"
+	case codes.DataLoss:
+		return "DATA_LOSS"
+	case codes.Unauthenticated:
+		return "UNAUTHENTICATED"
+	default:
+		return "CODE_" + strconv.FormatInt(int64(c), 10)
+	}
+}
diff --git a/pkg/metrics/grpcmetrics/grpc.go b/pkg/metrics/grpcmetrics/grpc.go
new file mode 100644
index 0000000..bed0e1b
--- /dev/null
+++ b/pkg/metrics/grpcmetrics/grpc.go
@@ -0,0 +1,65 @@
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcmetrics
+
+import (
+	"time"
+
+	"go.opencensus.io/tag"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/stats"
+)
+
+type ClientHandler struct {
+}
+
+func (c *ClientHandler) HandleConn(ctx context.Context, cs stats.ConnStats) {
+	// no-op
+}
+
+func (c *ClientHandler) TagConn(ctx context.Context, cti *stats.ConnTagInfo) context.Context {
+	// no-op
+	return ctx
+}
+
+func (c *ClientHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	statsHandleRPC(ctx, rs)
+}
+
+func (c *ClientHandler) TagRPC(ctx context.Context, rti *stats.RPCTagInfo) context.Context {
+	ctx = c.statsTagRPC(ctx, rti)
+	return ctx
+}
+
+func (h *ClientHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	startTime := time.Now()
+	if info == nil {
+		if grpclog.V(2) {
+			grpclog.Infoln("Failed to retrieve *rpcData from context.")
+		}
+		return ctx
+	}
+	d := &rpcData{
+		startTime: startTime,
+		method:    info.FullMethodName,
+	}
+	ts := tag.FromContext(ctx)
+	if ts != nil {
+		encoded := tag.Encode(ts)
+		ctx = stats.SetTags(ctx, encoded)
+	}
+	return context.WithValue(ctx, rpcDataKey, d)
+}
diff --git a/pkg/metrics/grpcmetrics/grpc_stats.go b/pkg/metrics/grpcmetrics/grpc_stats.go
new file mode 100644
index 0000000..6a75fab
--- /dev/null
+++ b/pkg/metrics/grpcmetrics/grpc_stats.go
@@ -0,0 +1,123 @@
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcmetrics
+
+import (
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+var (
+	ClientSentMessagesPerRPC = stats.Int64(
+		"grpc/client/sent_messages_per_rpc",
+		"Number of messages sent in the RPC (always 1 for non-streaming RPCs).",
+		stats.UnitDimensionless,
+	)
+
+	ClientSentBytesPerRPC = stats.Int64(
+		"grpc/client/sent_bytes_per_rpc",
+		"Total bytes sent across all request messages per RPC.",
+		stats.UnitBytes,
+	)
+
+	ClientReceivedMessagesPerRPC = stats.Int64(
+		"grpc/client/received_messages_per_rpc",
+		"Number of response messages received per RPC (always 1 for non-streaming RPCs).",
+		stats.UnitDimensionless,
+	)
+
+	ClientReceivedBytesPerRPC = stats.Int64(
+		"grpc/client/received_bytes_per_rpc",
+		"Total bytes received across all response messages per RPC.",
+		stats.UnitBytes,
+	)
+
+	ClientRoundtripLatency = stats.Float64(
+		"grpc/client/roundtrip_latency",
+		"Time between first byte of request sent to last byte of response received, or terminal error.",
+		stats.UnitMilliseconds,
+	)
+)
+
+var (
+	KeyClientMethod, _ = tag.NewKey("grpc_client_method")
+	KeyClientStatus, _ = tag.NewKey("grpc_client_status")
+)
+
+var (
+	ClientSentBytesPerRPCView = &view.View{
+		Measure:     ClientSentBytesPerRPC,
+		Name:        "grpc/client/sent_bytes_per_rpc",
+		Description: "Distribution of bytes sent per RPC, by method.",
+		TagKeys:     []tag.Key{KeyClientMethod},
+		Aggregation: DefaultBytesDistribution,
+	}
+
+	ClientReceivedBytesPerRPCView = &view.View{
+		Measure:     ClientReceivedBytesPerRPC,
+		Name:        "grpc/client/received_bytes_per_rpc",
+		Description: "Distribution of bytes received per RPC, by method.",
+		TagKeys:     []tag.Key{KeyClientMethod},
+		Aggregation: DefaultBytesDistribution,
+	}
+
+	ClientRoundtripLatencyView = &view.View{
+		Measure:     ClientRoundtripLatency,
+		Name:        "grpc/client/roundtrip_latency",
+		Description: "Distribution of round-trip latency, by method.",
+		TagKeys:     []tag.Key{KeyClientMethod},
+		Aggregation: DefaultMillisecondsDistribution,
+	}
+
+	ClientCompletedRPCsView = &view.View{
+		Measure:     ClientRoundtripLatency,
+		Name:        "grpc/client/completed_rpcs",
+		Description: "Count of RPCs by method and status.",
+		TagKeys:     []tag.Key{KeyClientMethod, KeyClientStatus},
+		Aggregation: view.Count(),
+	}
+
+	ClientSentMessagesPerRPCView = &view.View{
+		Measure:     ClientSentMessagesPerRPC,
+		Name:        "grpc/client/sent_messages_per_rpc",
+		Description: "Distribution of sent messages count per RPC, by method.",
+		TagKeys:     []tag.Key{KeyClientMethod},
+		Aggregation: DefaultMessageCountDistribution,
+	}
+
+	ClientReceivedMessagesPerRPCView = &view.View{
+		Measure:     ClientReceivedMessagesPerRPC,
+		Name:        "grpc/client/received_messages_per_rpc",
+		Description: "Distribution of received messages count per RPC, by method.",
+		TagKeys:     []tag.Key{KeyClientMethod},
+		Aggregation: DefaultMessageCountDistribution,
+	}
+)
+
+var DefaultClientViews = []*view.View{
+	ClientSentBytesPerRPCView,
+	ClientReceivedBytesPerRPCView,
+	ClientRoundtripLatencyView,
+	ClientCompletedRPCsView,
+	ClientSentMessagesPerRPCView,
+	ClientReceivedMessagesPerRPCView,
+}
+
+var (
+	DefaultBytesDistribution        = view.Distribution(0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296)
+	DefaultMillisecondsDistribution = view.Distribution(0, 0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000)
+	DefaultMessageCountDistribution = view.Distribution(0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536)
+)
diff --git a/pkg/metrics/httpmetrics/BUILD.bazel b/pkg/metrics/httpmetrics/BUILD.bazel
new file mode 100644
index 0000000..d73eea9
--- /dev/null
+++ b/pkg/metrics/httpmetrics/BUILD.bazel
@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "http.go",
+        "http_stats.go",
+    ],
+    importpath = "github.com/nghialv/lotus/pkg/metrics/httpmetrics",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@io_opencensus_go//stats:go_default_library",
+        "@io_opencensus_go//stats/view:go_default_library",
+        "@io_opencensus_go//tag:go_default_library",
+    ],
+)
diff --git a/pkg/metrics/httpmetrics/http.go b/pkg/metrics/httpmetrics/http.go
new file mode 100644
index 0000000..7a1d21a
--- /dev/null
+++ b/pkg/metrics/httpmetrics/http.go
@@ -0,0 +1,138 @@
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpmetrics
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+)
+
+type Transport struct {
+	Base           http.RoundTripper
+	UsePathAsRoute bool
+}
+
+func ContextWithRoute(ctx context.Context, route string) (context.Context, error) {
+	return tag.New(ctx, tag.Upsert(KeyClientRoute, route))
+}
+
+func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	mutators := []tag.Mutator{
+		tag.Upsert(KeyClientHost, req.URL.Host),
+		tag.Upsert(KeyClientMethod, req.Method),
+	}
+	if t.UsePathAsRoute {
+		mutators = append(mutators, tag.Upsert(KeyClientRoute, req.URL.Path))
+	} else {
+		mutators = append(mutators, tag.Insert(KeyClientRoute, "no_route"))
+	}
+	ctx, _ := tag.New(req.Context(), mutators...)
+	req = req.WithContext(ctx)
+	track := &tracker{
+		start: time.Now(),
+		ctx:   ctx,
+	}
+	track.reqSize = req.ContentLength
+	if req.Body == nil && req.ContentLength == -1 {
+		track.reqSize = 0
+	}
+
+	resp, err := t.base().RoundTrip(req)
+	if err != nil {
+		track.statusCode = http.StatusInternalServerError
+		track.end()
+	} else {
+		track.statusCode = resp.StatusCode
+		track.respContentLength = resp.ContentLength
+		if resp.Body == nil {
+			track.end()
+		} else {
+			track.body = resp.Body
+			resp.Body = track
+		}
+	}
+	return resp, err
+}
+
+func (t *Transport) CancelRequest(req *http.Request) {
+	type canceler interface {
+		CancelRequest(*http.Request)
+	}
+	if cr, ok := t.base().(canceler); ok {
+		cr.CancelRequest(req)
+	}
+}
+
+func (t *Transport) base() http.RoundTripper {
+	if t.Base != nil {
+		return t.Base
+	}
+	return http.DefaultTransport
+}
+
+type tracker struct {
+	ctx               context.Context
+	respSize          int64
+	respContentLength int64
+	reqSize           int64
+	start             time.Time
+	body              io.ReadCloser
+	statusCode        int
+	endOnce           sync.Once
+}
+
+var _ io.ReadCloser = (*tracker)(nil)
+
+func (t *tracker) end() {
+	t.endOnce.Do(func() {
+		latencyMs := float64(time.Since(t.start)) / float64(time.Millisecond)
+		respSize := t.respSize
+		if t.respSize == 0 && t.respContentLength > 0 {
+			respSize = t.respContentLength
+		}
+		m := []stats.Measurement{
+			ClientSentBytes.M(t.reqSize),
+			ClientReceivedBytes.M(respSize),
+			ClientRoundtripLatency.M(latencyMs),
+		}
+		stats.RecordWithTags(t.ctx, []tag.Mutator{
+			tag.Upsert(KeyClientStatus, strconv.Itoa(t.statusCode)),
+		}, m...)
+	})
+}
+
+func (t *tracker) Read(b []byte) (int, error) {
+	n, err := t.body.Read(b)
+	t.respSize += int64(n)
+	switch err {
+	case nil:
+		return n, nil
+	case io.EOF:
+		t.end()
+	}
+	return n, err
+}
+
+func (t *tracker) Close() error {
+	t.end()
+	return t.body.Close()
+}
diff --git a/pkg/metrics/httpmetrics/http_stats.go b/pkg/metrics/httpmetrics/http_stats.go
new file mode 100644
index 0000000..4fc367b
--- /dev/null
+++ b/pkg/metrics/httpmetrics/http_stats.go
@@ -0,0 +1,101 @@
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpmetrics
+
+import (
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+var (
+	ClientSentBytes = stats.Int64(
+		"http/client/sent_bytes",
+		"Total bytes sent in request body (not including headers)",
+		stats.UnitBytes,
+	)
+	ClientReceivedBytes = stats.Int64(
+		"http/client/received_bytes",
+		"Total bytes received in response bodies (not including headers but including error responses with bodies)",
+		stats.UnitBytes,
+	)
+	ClientRoundtripLatency = stats.Float64(
+		"http/client/roundtrip_latency",
+		"Time between first byte of request headers sent to last byte of response received, or terminal error",
+		stats.UnitMilliseconds,
+	)
+)
+
+// KeyClientRoute is a low cardinality string representing the logical
+// handler of the request. This is usually the pattern of the request.
+var (
+	KeyClientHost, _   = tag.NewKey("http_client_host")
+	KeyClientRoute, _  = tag.NewKey("http_client_route")
+	KeyClientMethod, _ = tag.NewKey("http_client_method")
+	KeyClientStatus, _ = tag.NewKey("http_client_status")
+
+	HTTPClientTagKeys = []tag.Key{
+		KeyClientHost,
+		KeyClientMethod,
+		KeyClientRoute,
+		KeyClientStatus,
+	}
+)
+
+var (
+	ClientSentBytesDistribution = &view.View{
+		Name:        "http/client/sent_bytes",
+		Measure:     ClientSentBytes,
+		Aggregation: DefaultSizeDistribution,
+		Description: "Total bytes sent in request body (not including headers)",
+		TagKeys:     HTTPClientTagKeys,
+	}
+
+	ClientReceivedBytesDistribution = &view.View{
+		Name:        "http/client/received_bytes",
+		Measure:     ClientReceivedBytes,
+		Aggregation: DefaultSizeDistribution,
+		Description: "Total bytes received in response bodies (not including headers but including error responses with bodies)",
+		TagKeys:     HTTPClientTagKeys,
+	}
+
+	ClientRoundtripLatencyDistribution = &view.View{
+		Name:        "http/client/roundtrip_latency",
+		Measure:     ClientRoundtripLatency,
+		Aggregation: DefaultLatencyDistribution,
+		Description: "End-to-end latency",
+		TagKeys:     HTTPClientTagKeys,
+	}
+
+	ClientCompletedCount = &view.View{
+		Name:        "http/client/completed_count",
+		Measure:     ClientRoundtripLatency,
+		Aggregation: view.Count(),
+		Description: "Count of completed requests",
+		TagKeys:     HTTPClientTagKeys,
+	}
+)
+
+var DefaultClientViews = []*view.View{
+	ClientCompletedCount,
+	ClientSentBytesDistribution,
+	ClientReceivedBytesDistribution,
+	ClientRoundtripLatencyDistribution,
+}
+
+var (
+	DefaultSizeDistribution    = view.Distribution(0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296)
+	DefaultLatencyDistribution = view.Distribution(0, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000)
+)
diff --git a/pkg/metrics/logger.go b/pkg/metrics/logger.go
new file mode 100644
index 0000000..b5083d5
--- /dev/null
+++ b/pkg/metrics/logger.go
@@ -0,0 +1,18 @@
+package metrics
+
+import "log"
+
+type Logger interface {
+	Infof(format string, args ...interface{})
+	Errorf(format string, args ...interface{})
+}
+
+type logger struct{}
+
+func (l logger) Infof(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+func (l logger) Errorf(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
new file mode 100644
index 0000000..5e8c0ab
--- /dev/null
+++ b/pkg/metrics/metrics.go
@@ -0,0 +1,156 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"go.opencensus.io/exporter/prometheus"
+	"go.opencensus.io/stats/view"
+
+	"github.com/nghialv/lotus/pkg/metrics/grpcmetrics"
+	"github.com/nghialv/lotus/pkg/metrics/httpmetrics"
+	"github.com/nghialv/lotus/pkg/virtualuser"
+)
+
+type options struct {
+	namespace        string
+	path             string
+	reportingPeriod  time.Duration
+	gracefulPeriod   time.Duration
+	grpcViews        []*view.View
+	httpViews        []*view.View
+	virtualUserViews []*view.View
+	customViews      []*view.View
+	logger           Logger
+}
+
+var defaultOptions = options{
+	namespace:       "lotus",
+	path:            "/metrics",
+	reportingPeriod: time.Second,
+	gracefulPeriod:  5 * time.Second,
+	grpcViews:       grpcmetrics.DefaultClientViews,
+	httpViews:       httpmetrics.DefaultClientViews,
+	virtualUserViews: []*view.View{
+		virtualuser.UserCountView,
+	},
+	logger: logger{},
+}
+
+type Option func(*options)
+
+func WithNamespace(namespace string) Option {
+	return func(opts *options) {
+		opts.namespace = namespace
+	}
+}
+
+func WithPath(path string) Option {
+	return func(opts *options) {
+		opts.path = path
+	}
+}
+
+func WithReportingPeriod(period time.Duration) Option {
+	return func(opts *options) {
+		opts.reportingPeriod = period
+	}
+}
+
+func WithGracefulPeriod(period time.Duration) Option {
+	return func(opts *options) {
+		opts.gracefulPeriod = period
+	}
+}
+
+func WithCustomViews(views ...*view.View) Option {
+	return func(opts *options) {
+		opts.customViews = views
+	}
+}
+
+func WithGrpcViews(views ...*view.View) Option {
+	return func(opts *options) {
+		opts.grpcViews = views
+	}
+}
+
+func WithHttpViews(views ...*view.View) Option {
+	return func(opts *options) {
+		opts.httpViews = views
+	}
+}
+
+func WithLogger(logger Logger) Option {
+	return func(opts *options) {
+		opts.logger = logger
+	}
+}
+
+func (o options) Views() []*view.View {
+	length := len(o.grpcViews) + len(o.httpViews) + len(o.virtualUserViews) + len(o.customViews)
+	views := make([]*view.View, 0, length)
+	views = append(views, o.grpcViews...)
+	views = append(views, o.httpViews...)
+	views = append(views, o.virtualUserViews...)
+	views = append(views, o.customViews...)
+	return views
+}
+
+type server struct {
+	server *http.Server
+	opts   options
+}
+
+func NewServer(port int, opt ...Option) (*server, error) {
+	opts := defaultOptions
+	for _, o := range opt {
+		o(&opts)
+	}
+	view.SetReportingPeriod(opts.reportingPeriod)
+	err := view.Register(opts.Views()...)
+	if err != nil {
+		opts.logger.Errorf("failed to register views: %v", err)
+		return nil, err
+	}
+	pe, err := prometheus.NewExporter(prometheus.Options{
+		Namespace: opts.namespace,
+	})
+	if err != nil {
+		opts.logger.Errorf("failed to create prometheus exporter: %v", err)
+		return nil, err
+	}
+	view.RegisterExporter(pe)
+
+	mux := http.NewServeMux()
+	mux.Handle(opts.path, pe)
+	s := &http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: mux,
+	}
+	return &server{
+		server: s,
+		opts:   opts,
+	}, nil
+}
+
+func (s *server) Run() error {
+	err := s.server.ListenAndServe()
+	if err != nil && err != http.ErrServerClosed {
+		s.opts.logger.Errorf("failed to run metrics server: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (s *server) Stop() error {
+	ctx, cancel := context.WithTimeout(context.Background(), s.opts.gracefulPeriod)
+	defer cancel()
+	err := s.server.Shutdown(ctx)
+	if err != nil {
+		s.opts.logger.Errorf("failed to shutdown metrics server: %v", err)
+	}
+	return err
+}
diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go
new file mode 100644
index 0000000..1abe097
--- /dev/null
+++ b/pkg/metrics/metrics_test.go
@@ -0,0 +1 @@
+package metrics
diff --git a/pkg/version/BUILD.bazel b/pkg/version/BUILD.bazel
new file mode 100644
index 0000000..dff254b
--- /dev/null
+++ b/pkg/version/BUILD.bazel
@@ -0,0 +1,10 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//pkg/version:def.bzl", "version_x_defs")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["version.go"],
+    importpath = "github.com/nghialv/lotus/pkg/version",
+    visibility = ["//visibility:public"],
+    x_defs = version_x_defs(),
+)
diff --git a/pkg/version/def.bzl b/pkg/version/def.bzl
new file mode 100644
index 0000000..85dfa72
--- /dev/null
+++ b/pkg/version/def.bzl
@@ -0,0 +1,35 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See https://github.com/nghialv/lotus/tree/master/NOTICE.md
+
+def version_x_defs():
+    stamp_pkgs = [
+        "github.com/nghialv/lotus/pkg/version",
+    ]
+
+    # This should match the list of vars set in hack/print-workspace-status.sh.
+    stamp_vars = [
+        "gitCommit",
+        "gitCommitFull",
+        "buildDate",
+        "version",
+    ]
+
+    # Generate the cross-product.
+    x_defs = {}
+    for pkg in stamp_pkgs:
+        for var in stamp_vars:
+            x_defs["%s.%s" % (pkg, var)] = "{%s}" % var
+    return x_defs
diff --git a/pkg/version/version.go b/pkg/version/version.go
new file mode 100644
index 0000000..9b4ecd6
--- /dev/null
+++ b/pkg/version/version.go
@@ -0,0 +1,36 @@
+package version
+
+import "fmt"
+
+var (
+	gitCommit     = "unspecified"
+	gitCommitFull = "unspecified"
+	buildDate     = "unspecified"
+	version       = "unspecified"
+)
+
+type Info struct {
+	GitCommit     string
+	GitCommitFull string
+	BuildDate     string
+	Version       string
+}
+
+func Get() Info {
+	return Info{
+		GitCommit:     gitCommit,
+		GitCommitFull: gitCommitFull,
+		BuildDate:     buildDate,
+		Version:       version,
+	}
+}
+
+func (i Info) String() string {
+	return fmt.Sprintf(
+		"Version: %s, GitCommit: %s, GitCommitFull: %s, BuildDate: %s",
+		i.Version,
+		i.GitCommit,
+		i.GitCommitFull,
+		i.BuildDate,
+	)
+}
diff --git a/pkg/virtualuser/BUILD.bazel b/pkg/virtualuser/BUILD.bazel
new file mode 100644
index 0000000..900764b
--- /dev/null
+++ b/pkg/virtualuser/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["virtualuser.go"],
+    importpath = "github.com/nghialv/lotus/pkg/virtualuser",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@io_opencensus_go//stats:go_default_library",
+        "@io_opencensus_go//stats/view:go_default_library",
+        "@io_opencensus_go//tag:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["virtualuser_test.go"],
+    embed = [":go_default_library"],
+)
diff --git a/pkg/virtualuser/virtualuser.go b/pkg/virtualuser/virtualuser.go
new file mode 100644
index 0000000..0114b13
--- /dev/null
+++ b/pkg/virtualuser/virtualuser.go
@@ -0,0 +1,121 @@
+package virtualuser
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+type (
+	VirtualUser interface {
+		Run(ctx context.Context) error
+	}
+
+	Factory func() (VirtualUser, error)
+)
+
+const (
+	StatusStarted   = "started"
+	StatusSucceeded = "succeeded"
+	StatusFailed    = "failed"
+)
+
+var (
+	UserCount = stats.Int64("virtual_user/count", "Number of virtual users", stats.UnitDimensionless)
+
+	KeyStatus, _ = tag.NewKey("virtual_user_status")
+
+	UserCountView = &view.View{
+		Name:        "virtual_user/count",
+		Measure:     UserCount,
+		TagKeys:     []tag.Key{KeyStatus},
+		Description: "Count of virtual users by status",
+		Aggregation: view.Count(),
+	}
+)
+
+type Group struct {
+	numUsers  int
+	hatchRate int
+	factory   Factory
+	doneCh    chan struct{}
+}
+
+func NewGroup(numUsers, hatchRate int, factory Factory) *Group {
+	return &Group{
+		numUsers:  numUsers,
+		hatchRate: hatchRate,
+		factory:   factory,
+		doneCh:    make(chan struct{}),
+	}
+}
+
+func (g *Group) Run(ctx context.Context) error {
+	startedCtx, err := tag.New(ctx, tag.Insert(KeyStatus, StatusStarted))
+	if err != nil {
+		return err
+	}
+	succeededCtx, err := tag.New(ctx, tag.Insert(KeyStatus, StatusSucceeded))
+	if err != nil {
+		return err
+	}
+	failedCtx, err := tag.New(ctx, tag.Insert(KeyStatus, StatusFailed))
+	if err != nil {
+		return err
+	}
+	var index int = 0
+	var wg sync.WaitGroup
+	for index < g.numUsers {
+		for i := 0; i < g.hatchRate && index < g.numUsers; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				stats.Record(startedCtx, UserCount.M(1))
+				var err error
+				defer func() {
+					if r := recover(); r != nil {
+						err = errors.New("virtualuser.group: panic")
+					}
+					if err != nil {
+						stats.Record(succeededCtx, UserCount.M(1))
+					} else {
+						stats.Record(failedCtx, UserCount.M(1))
+					}
+				}()
+				var vu VirtualUser
+				vu, err = g.factory()
+				if err != nil {
+					return
+				}
+				err = vu.Run(ctx)
+			}()
+			index++
+		}
+		if index >= g.numUsers {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			break
+		case <-time.After(time.Second):
+		}
+	}
+	wg.Wait()
+	close(g.doneCh)
+	return nil
+}
+
+func (g *Group) Stop(timeout time.Duration) error {
+	select {
+	case <-g.doneCh:
+		return nil
+	case <-time.After(timeout):
+		return fmt.Errorf("timed out")
+	}
+}
diff --git a/pkg/virtualuser/virtualuser_test.go b/pkg/virtualuser/virtualuser_test.go
new file mode 100644
index 0000000..e706c05
--- /dev/null
+++ b/pkg/virtualuser/virtualuser_test.go
@@ -0,0 +1 @@
+package virtualuser