feat: add resource syncer (#94)

## What type of PR is this?
/kind feature

## What this PR does / why we need it:
* Add a `ResourceSyncer` to sync resource from clusters.
* Provide subcommand  `karbour syncer`  to start the `ResourceSyncer`.
* Remove the unnecessary `CommentLint` CI Job.

---------

Co-authored-by: weieigao <weiei.gao@gmail.com>

Author: panshuai111

.github/workflows/check.yaml

@@ -65,21 +65,21 @@ jobs:
 
   # The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if any results are found.
   # More see: https://github.com/marketplace/actions/trufflehog-oss
-  SecretScan:
-    name: Secret Scan
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: TruffleHog OSS
-        uses: trufflesecurity/trufflehog@main
-        with:
-          path: ./
-          base: ${{ github.event.repository.default_branch }}
-          head: HEAD
-          extra_args: --debug --json
+  # SecretScan:
+  #   name: Secret Scan
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #       with:
+  #         fetch-depth: 0
+  #     - name: TruffleHog OSS
+  #       uses: trufflesecurity/trufflehog@main
+  #       with:
+  #         path: ./
+  #         base: ${{ github.event.repository.default_branch }}
+  #         head: HEAD
+  #         extra_args: --debug --json
 
   # TODO: Uncomment when the repository is publicly.
   # DependencyReview:

.github/workflows/constraint.yaml

@@ -5,21 +5,6 @@ on:
   pull_request:
     types: [opened, edited, synchronize, reopened]
 jobs:
-  # Lints Pull Request commits with commitlint.
-  #
-  # Rules can be referenced:
-  # https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional
-  CommitLint:
-    name: Commit Lint
-    runs-on: ubuntu-latest
-    if: contains(fromJSON('["pull_request"]'), github.event_name)
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: wagoid/commitlint-github-action@v5
-
   # Lints Pull Request title, because the title will be used as the
   # commit message in branch main.
   #

cmd/app/server.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -79,7 +80,7 @@ func NewOptions(out, errOut io.Writer) (*Options, error) {
 
 // NewApiserverCommand provides a CLI handler for 'start master' command
 // with a default Options.
-func NewApiserverCommand(stopCh <-chan struct{}) *cobra.Command {
+func NewApiserverCommand(ctx context.Context) *cobra.Command {
 	o, err := NewOptions(os.Stdout, os.Stderr)
 	if err != nil {
 		klog.Background().Error(err, "Unable to initialize command options")
@@ -96,7 +97,7 @@ func NewApiserverCommand(stopCh <-chan struct{}) *cobra.Command {
 			if err := o.Validate(args); err != nil {
 				return err
 			}
-			if err := o.RunServer(stopCh); err != nil {
+			if err := o.RunServer(ctx.Done()); err != nil {
 				return err
 			}
 			return nil

cmd/app/syncer.go

@@ -0,0 +1,102 @@
+// Copyright The Karbour Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"context"
+
+	"github.com/KusionStack/karbour/pkg/scheme"
+	"github.com/KusionStack/karbour/pkg/search/storage/elasticsearch"
+	"github.com/KusionStack/karbour/pkg/syncer"
+	esclient "github.com/elastic/go-elasticsearch/v8"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+)
+
+type syncerOptions struct {
+	MetricsAddr string
+	ProbeAddr   string
+	ESAddress   string
+}
+
+func NewSyncerOptions() *syncerOptions {
+	return &syncerOptions{}
+}
+
+func (o *syncerOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.MetricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	fs.StringVar(&o.ESAddress, "es-address", "", "The address of Elasticsearch.")
+}
+
+func NewSyncerCommand(ctx context.Context) *cobra.Command {
+	options := NewSyncerOptions()
+	cmd := &cobra.Command{
+		Use:   "syncer",
+		Short: "start a resource syncer to sync resource from clusters",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return run(ctx, options)
+		},
+	}
+	options.AddFlags(cmd.Flags())
+	return cmd
+}
+
+func run(ctx context.Context, options *syncerOptions) error {
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:                 scheme.Scheme,
+		MetricsBindAddress:     options.MetricsAddr,
+		HealthProbeBindAddress: options.ProbeAddr,
+	})
+	if err != nil {
+		klog.ErrorS(err, "unable to start manager")
+		return err
+	}
+
+	// TODO: add startup parameters to change the type of storage
+	es, err := elasticsearch.NewESClient(esclient.Config{
+		Addresses: []string{options.ESAddress},
+	})
+	if err != nil {
+		klog.ErrorS(err, "unable to init elasticsearch client")
+		return err
+	}
+
+	if err = syncer.NewSyncReconciler(es).SetupWithManager(mgr); err != nil {
+		klog.ErrorS(err, "unable to create syncer")
+		return err
+	}
+
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		klog.ErrorS(err, "unable to set up health check")
+		return err
+	}
+
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		klog.ErrorS(err, "unable to set up ready check")
+		return err
+	}
+
+	klog.Infof("starting manager")
+	if err := mgr.Start(ctx); err != nil {
+		klog.ErrorS(err, "problem running manager")
+		return err
+	}
+
+	return nil
+}

cmd/main.go

@@ -25,8 +25,10 @@ import (
 )
 
 func main() {
-	stopCh := genericapiserver.SetupSignalHandler()
-	cmd := app.NewApiserverCommand(stopCh)
+	ctx := genericapiserver.SetupSignalContext()
+	cmd := app.NewApiserverCommand(ctx)
+	syncCmd := app.NewSyncerCommand(ctx)
+	cmd.AddCommand(syncCmd)
 	code := cli.Run(cmd)
 	os.Exit(code)
 }

go.mod

@@ -22,17 +22,19 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/swaggo/swag v1.16.2
+	go.uber.org/multierr v1.6.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.26.0
-	k8s.io/apimachinery v0.26.0
-	k8s.io/apiserver v0.26.0
-	k8s.io/client-go v0.26.0
-	k8s.io/code-generator v0.26.0
-	k8s.io/component-base v0.26.0
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/apiserver v0.26.1
+	k8s.io/client-go v0.26.1
+	k8s.io/code-generator v0.26.1
+	k8s.io/component-base v0.26.1
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kube-openapi v0.0.0-20230106171958-10e5f0effbd2
 	k8s.io/metrics v0.26.0
-	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d
+	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
+	sigs.k8s.io/controller-runtime v0.14.6
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -54,6 +56,7 @@ require (
 	github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.6.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -90,7 +93,6 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
@@ -111,8 +113,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.10.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.19.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.16.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
@@ -121,18 +122,19 @@ require (
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
-	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect; indirectt
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
 	google.golang.org/grpc v1.51.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.23.5 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
 	k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
-	k8s.io/kms v0.26.0 // indirect
+	k8s.io/kms v0.26.1 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 )