From 5ee72afcff42b6f613f1597fa8b56958f0f49b55 Mon Sep 17 00:00:00 2001 From: AdheipSingh Date: Sat, 29 Apr 2023 16:53:13 +0530 Subject: [PATCH 1/2] add basic auth support in cp --- Makefile | 2 +- api/v1beta1/pinot_types.go | 15 + api/v1beta1/zz_generated.deepcopy.go | 17 + config/crd/bases/datainfra.io_pinots.yaml | 22 ++ config/crd/kustomization.yaml | 2 +- .../patches/cainjection_in_pinottenants.yaml | 2 +- config/rbac/pinotschema_editor_role.yaml | 6 + config/rbac/pinottable_editor_role.yaml | 8 + config/rbac/pinottenant_viewer_role.yaml | 12 +- docs/getting_started_auth.md | 53 +++ examples/04-pinot-auth/pinotauth-basic.yaml | 8 +- examples/04-pinot-auth/pinotauth-schema.yaml | 356 ++++++++++++++++++ examples/04-pinot-auth/pinotauth-table.yaml | 44 +++ .../{pinotbasic-aws.yaml => pinot-aws.yaml} | 0 helm/pinot-control-plane/Chart.yaml | 4 +- .../templates/crds/datainfra.io_pinots.yaml | 22 ++ .../templates/rbac_manager.yaml | 8 + internal/http/http.go | 16 +- .../pinotschema_controller.go | 1 + internal/schema_controller/reconciler.go | 87 ++++- .../table_controller/pinottable_controller.go | 8 +- internal/table_controller/reconciler.go | 81 +++- .../pinottenant_controller.go | 8 +- internal/tenant_controller/reconciler.go | 85 ++++- 24 files changed, 831 insertions(+), 36 deletions(-) create mode 100644 docs/getting_started_auth.md create mode 100644 examples/04-pinot-auth/pinotauth-schema.yaml create mode 100644 examples/04-pinot-auth/pinotauth-table.yaml rename examples/05-pinot-aws/{pinotbasic-aws.yaml => pinot-aws.yaml} (100%) diff --git a/Makefile b/Makefile index 0afeb10..2a3c7cb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= datainfrahq/pinot-control-plane:v0.0.6 +IMG ?= datainfrahq/pinot-control-plane:v0.0.7 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.26.0 diff --git a/api/v1beta1/pinot_types.go b/api/v1beta1/pinot_types.go index bba064c..cb8b2db 100644 --- a/api/v1beta1/pinot_types.go +++ b/api/v1beta1/pinot_types.go @@ -23,6 +23,8 @@ import ( // PinotSpec defines the desired state of Pinot type PinotSpec struct { + // +optional + Auth Auth `json:"auth"` // +optional Plugins []string `json:"plugins"` // +required @@ -66,6 +68,19 @@ type DeepStorageConfig struct { Data string `json:"data"` } +type AuthType string + +const ( + BasicAuth AuthType = "basic-auth" +) + +type Auth struct { + // +required + Type AuthType `json:"type"` + // +required + SecretRef v1.SecretReference `json:"secretRef"` +} + type K8sConfig struct { // +required Name string `json:"name"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 12e5687..b472187 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -26,6 +26,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Auth) DeepCopyInto(out *Auth) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Auth. +func (in *Auth) DeepCopy() *Auth { + if in == nil { + return nil + } + out := new(Auth) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeepStorageConfig) DeepCopyInto(out *DeepStorageConfig) { *out = *in @@ -372,6 +388,7 @@ func (in *PinotSchemaStatus) DeepCopy() *PinotSchemaStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PinotSpec) DeepCopyInto(out *PinotSpec) { *out = *in + out.Auth = in.Auth if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = make([]string, len(*in)) diff --git a/config/crd/bases/datainfra.io_pinots.yaml b/config/crd/bases/datainfra.io_pinots.yaml index ac7960d..34398c9 100644 --- a/config/crd/bases/datainfra.io_pinots.yaml +++ b/config/crd/bases/datainfra.io_pinots.yaml @@ -39,6 +39,28 @@ spec: spec: description: PinotSpec defines the desired state of Pinot properties: + auth: + properties: + secretRef: + description: SecretReference represents a Secret Reference. It + has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + type: + type: string + required: + - secretRef + - type + type: object deploymentOrder: items: type: string diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5e48e39..9a1ff45 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,7 +5,7 @@ resources: - bases/datainfra.io_pinots.yaml - bases/datainfra.io_pinotschemas.yaml - bases/datainfra.io_pinottables.yaml -- bases/datainfra.io.datainfra.io_pinottenants.yaml +- bases/datainfra.io_pinottenants.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/crd/patches/cainjection_in_pinottenants.yaml b/config/crd/patches/cainjection_in_pinottenants.yaml index a5c06fd..19fea8f 100644 --- a/config/crd/patches/cainjection_in_pinottenants.yaml +++ b/config/crd/patches/cainjection_in_pinottenants.yaml @@ -4,4 +4,4 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME - name: pinottenants.datainfra.io.datainfra.io + name: pinottenants.datainfra.io diff --git a/config/rbac/pinotschema_editor_role.yaml b/config/rbac/pinotschema_editor_role.yaml index 05792f0..1a4c099 100644 --- a/config/rbac/pinotschema_editor_role.yaml +++ b/config/rbac/pinotschema_editor_role.yaml @@ -23,6 +23,12 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - datainfra.io resources: diff --git a/config/rbac/pinottable_editor_role.yaml b/config/rbac/pinottable_editor_role.yaml index 5361570..8af04b6 100644 --- a/config/rbac/pinottable_editor_role.yaml +++ b/config/rbac/pinottable_editor_role.yaml @@ -23,6 +23,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch - apiGroups: - datainfra.io resources: diff --git a/config/rbac/pinottenant_viewer_role.yaml b/config/rbac/pinottenant_viewer_role.yaml index d186af4..8f1b138 100644 --- a/config/rbac/pinottenant_viewer_role.yaml +++ b/config/rbac/pinottenant_viewer_role.yaml @@ -12,7 +12,7 @@ metadata: name: pinottenant-viewer-role rules: - apiGroups: - - datainfra.io.datainfra.io + - datainfra.io resources: - pinottenants verbs: @@ -20,7 +20,15 @@ rules: - list - watch - apiGroups: - - datainfra.io.datainfra.io + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - datainfra.io resources: - pinottenants/status verbs: diff --git a/docs/getting_started_auth.md b/docs/getting_started_auth.md new file mode 100644 index 0000000..f3ccefd --- /dev/null +++ b/docs/getting_started_auth.md @@ -0,0 +1,53 @@ +### Getting Started With Auth Enabled Cluster + +- Control Plane supports basic auth only. + +#### Export your StorageClassName +``` +export STORAGE_CLASS_NAME=standard +``` + +#### Install Pinot Control Plane +``` +make helm-install-pinot-control-plane +``` + +#### Install Zookeeper Opoerator and CR +``` +make helm-install-zk-operator +``` + +### Install Pinot Cluster + +``` +envsubst < examples/04-pinot-auth/pinotauth-basic.yaml | kubectl apply -f - -n pinot +``` + +### Create a K8 secret in the namespace where pinot cluster is deployed + + +- add secrets to file +``` +cat << EOF > pinot-control-plane-secret +CONTROL_PLANE_USERNAME=controlplane +CONTROL_PLANE_PASSWORD=controlplane +EOF +``` + +- create secret + +``` +kubectl create secret generic pinot-control-plane-secret --from-env-file=pinot-control-plane-secret -n pinot +``` + +### create schema + +``` +kubectl apply -f examples/04-pinot-auth/pinotauth-schema.yaml -n pinot +``` + +### create table + +``` +kubectl apply -f examples/04-pinot-auth/pinotauth-table.yaml -n pinot +``` diff --git a/examples/04-pinot-auth/pinotauth-basic.yaml b/examples/04-pinot-auth/pinotauth-basic.yaml index 0f167a3..67927d0 100644 --- a/examples/04-pinot-auth/pinotauth-basic.yaml +++ b/examples/04-pinot-auth/pinotauth-basic.yaml @@ -1,9 +1,15 @@ apiVersion: datainfra.io/v1beta1 kind: Pinot metadata: - name: pinot-basic + name: pinot-auth spec: + auth: + type: basic-auth + secretRef: + name: pinot-control-plane-secret + namespace: pinot + external: zookeeper: diff --git a/examples/04-pinot-auth/pinotauth-schema.yaml b/examples/04-pinot-auth/pinotauth-schema.yaml new file mode 100644 index 0000000..1df41b9 --- /dev/null +++ b/examples/04-pinot-auth/pinotauth-schema.yaml @@ -0,0 +1,356 @@ +apiVersion: datainfra.io/v1beta1 +kind: PinotSchema +metadata: + name: airlinestats +spec: + pinotCluster: pinot-auth + schema.json: |- + { + "metricFieldSpecs":[ + + ], + "dimensionFieldSpecs":[ + { + "dataType":"INT", + "name":"ActualElapsedTime" + }, + { + "dataType":"INT", + "name":"AirTime" + }, + { + "dataType":"INT", + "name":"AirlineID" + }, + { + "dataType":"INT", + "name":"ArrDel15" + }, + { + "dataType":"INT", + "name":"ArrDelay" + }, + { + "dataType":"INT", + "name":"ArrDelayMinutes" + }, + { + "dataType":"INT", + "name":"ArrTime" + }, + { + "dataType":"STRING", + "name":"ArrTimeBlk" + }, + { + "dataType":"INT", + "name":"ArrivalDelayGroups" + }, + { + "dataType":"INT", + "name":"CRSArrTime" + }, + { + "dataType":"INT", + "name":"CRSDepTime" + }, + { + "dataType":"INT", + "name":"CRSElapsedTime" + }, + { + "dataType":"STRING", + "name":"CancellationCode" + }, + { + "dataType":"INT", + "name":"Cancelled" + }, + { + "dataType":"STRING", + "name":"Carrier" + }, + { + "dataType":"INT", + "name":"CarrierDelay" + }, + { + "dataType":"INT", + "name":"DayOfWeek" + }, + { + "dataType":"INT", + "name":"DayofMonth" + }, + { + "dataType":"INT", + "name":"DepDel15" + }, + { + "dataType":"INT", + "name":"DepDelay" + }, + { + "dataType":"INT", + "name":"DepDelayMinutes" + }, + { + "dataType":"INT", + "name":"DepTime" + }, + { + "dataType":"STRING", + "name":"DepTimeBlk" + }, + { + "dataType":"INT", + "name":"DepartureDelayGroups" + }, + { + "dataType":"STRING", + "name":"Dest" + }, + { + "dataType":"INT", + "name":"DestAirportID" + }, + { + "dataType":"INT", + "name":"DestAirportSeqID" + }, + { + "dataType":"INT", + "name":"DestCityMarketID" + }, + { + "dataType":"STRING", + "name":"DestCityName" + }, + { + "dataType":"STRING", + "name":"DestState" + }, + { + "dataType":"INT", + "name":"DestStateFips" + }, + { + "dataType":"STRING", + "name":"DestStateName" + }, + { + "dataType":"INT", + "name":"DestWac" + }, + { + "dataType":"INT", + "name":"Distance" + }, + { + "dataType":"INT", + "name":"DistanceGroup" + }, + { + "dataType":"INT", + "name":"DivActualElapsedTime" + }, + { + "dataType":"INT", + "name":"DivAirportIDs", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivAirportLandings" + }, + { + "dataType":"INT", + "name":"DivAirportSeqIDs", + "singleValueField":false + }, + { + "dataType":"STRING", + "name":"DivAirports", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivArrDelay" + }, + { + "dataType":"INT", + "name":"DivDistance" + }, + { + "dataType":"INT", + "name":"DivLongestGTimes", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivReachedDest" + }, + { + "dataType":"STRING", + "name":"DivTailNums", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivTotalGTimes", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivWheelsOffs", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"DivWheelsOns", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"Diverted" + }, + { + "dataType":"INT", + "name":"FirstDepTime" + }, + { + "dataType":"STRING", + "name":"FlightDate" + }, + { + "dataType":"INT", + "name":"FlightNum" + }, + { + "dataType":"INT", + "name":"Flights" + }, + { + "dataType":"INT", + "name":"LateAircraftDelay" + }, + { + "dataType":"INT", + "name":"LongestAddGTime" + }, + { + "dataType":"INT", + "name":"Month" + }, + { + "dataType":"INT", + "name":"NASDelay" + }, + { + "dataType":"STRING", + "name":"Origin" + }, + { + "dataType":"INT", + "name":"OriginAirportID" + }, + { + "dataType":"INT", + "name":"OriginAirportSeqID" + }, + { + "dataType":"INT", + "name":"OriginCityMarketID" + }, + { + "dataType":"STRING", + "name":"OriginCityName" + }, + { + "dataType":"STRING", + "name":"OriginState" + }, + { + "dataType":"INT", + "name":"OriginStateFips" + }, + { + "dataType":"STRING", + "name":"OriginStateName" + }, + { + "dataType":"INT", + "name":"OriginWac" + }, + { + "dataType":"INT", + "name":"Quarter" + }, + { + "dataType":"STRING", + "name":"RandomAirports", + "singleValueField":false + }, + { + "dataType":"INT", + "name":"SecurityDelay" + }, + { + "dataType":"STRING", + "name":"TailNum" + }, + { + "dataType":"INT", + "name":"TaxiIn" + }, + { + "dataType":"INT", + "name":"TaxiOut" + }, + { + "dataType":"INT", + "name":"Year" + }, + { + "dataType":"INT", + "name":"WheelsOn" + }, + { + "dataType":"INT", + "name":"WheelsOff" + }, + { + "dataType":"INT", + "name":"WeatherDelay" + }, + { + "dataType":"STRING", + "name":"UniqueCarrier" + }, + { + "dataType":"INT", + "name":"TotalAddGTime" + } + ], + "dateTimeFieldSpecs":[ + { + "name":"DaysSinceEpoch", + "dataType":"INT", + "format":"1:DAYS:EPOCH", + "granularity":"1:DAYS" + }, + { + "name":"ts", + "dataType":"TIMESTAMP", + "format":"1:MILLISECONDS:TIMESTAMP", + "granularity":"1:SECONDS" + }, + { + "name":"tsRaw", + "dataType":"TIMESTAMP", + "format":"1:MILLISECONDS:TIMESTAMP", + "granularity":"1:SECONDS" + } + ], + "schemaName":"airlineStats" + } diff --git a/examples/04-pinot-auth/pinotauth-table.yaml b/examples/04-pinot-auth/pinotauth-table.yaml new file mode 100644 index 0000000..7ad6006 --- /dev/null +++ b/examples/04-pinot-auth/pinotauth-table.yaml @@ -0,0 +1,44 @@ +apiVersion: datainfra.io/v1beta1 +kind: PinotTable +metadata: + name: airlinestats +spec: + pinotCluster: pinot-auth + pinotSchema: airlinestats + pinotTableType: REALTIME + tables.json: |- + { + "tableName": "airlineStats", + "tableType": "REALTIME", + "segmentsConfig": { + "timeColumnName": "DaysSinceEpoch", + "timeType": "DAYS", + "retentionTimeUnit": "DAYS", + "retentionTimeValue": "5", + "segmentPushType": "APPEND", + "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", + "schemaName": "airlineStats", + "replication": "1", + "replicasPerPartition": "1" + }, + "tenants": {}, + "tableIndexConfig": { + "loadMode": "MMAP", + "streamConfigs": { + "streamType": "kafka", + "stream.kafka.consumer.type": "simple", + "stream.kafka.topic.name": "flights-realtime", + "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.hlc.zk.connect.string": "kafka-zookeeper:2181", + "stream.kafka.zk.broker.url": "kafka-zookeeper:2181", + "stream.kafka.broker.list": "kafka-0.kafka-headless.pinot.svc.cluster.local:9092", + "realtime.segment.flush.threshold.time": "3600000", + "realtime.segment.flush.threshold.size": "50000", + "stream.kafka.consumer.prop.auto.offset.reset": "smallest" + } + }, + "metadata": { + "customConfigs": {} + } + } diff --git a/examples/05-pinot-aws/pinotbasic-aws.yaml b/examples/05-pinot-aws/pinot-aws.yaml similarity index 100% rename from examples/05-pinot-aws/pinotbasic-aws.yaml rename to examples/05-pinot-aws/pinot-aws.yaml diff --git a/helm/pinot-control-plane/Chart.yaml b/helm/pinot-control-plane/Chart.yaml index 0962dc3..5cf7c94 100644 --- a/helm/pinot-control-plane/Chart.yaml +++ b/helm/pinot-control-plane/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: pinot-control-plane description: A Helm chart for Kubernetes type: application -version: 0.0.6 -appVersion: "v0.0.6" +version: 0.0.7 +appVersion: "v0.0.7" diff --git a/helm/pinot-control-plane/templates/crds/datainfra.io_pinots.yaml b/helm/pinot-control-plane/templates/crds/datainfra.io_pinots.yaml index ac7960d..34398c9 100644 --- a/helm/pinot-control-plane/templates/crds/datainfra.io_pinots.yaml +++ b/helm/pinot-control-plane/templates/crds/datainfra.io_pinots.yaml @@ -39,6 +39,28 @@ spec: spec: description: PinotSpec defines the desired state of Pinot properties: + auth: + properties: + secretRef: + description: SecretReference represents a Secret Reference. It + has enough information to retrieve secret in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + type: + type: string + required: + - secretRef + - type + type: object deploymentOrder: items: type: string diff --git a/helm/pinot-control-plane/templates/rbac_manager.yaml b/helm/pinot-control-plane/templates/rbac_manager.yaml index 5340241..0ba0268 100644 --- a/helm/pinot-control-plane/templates/rbac_manager.yaml +++ b/helm/pinot-control-plane/templates/rbac_manager.yaml @@ -228,6 +228,14 @@ rules: - get - patch - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch - apiGroups: - datainfra.io resources: diff --git a/internal/http/http.go b/internal/http/http.go index f21dbad..f3c16e0 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -31,20 +31,30 @@ type Client struct { URL string HTTPClient http.Client Body []byte + Auth Auth } +type Auth struct { + BasicAuth BasicAuth +} + +type BasicAuth struct { + UserName string + Password string +} type HttpResponse struct { RespBody []byte Err error StatusCode int } -func NewHTTPClient(method, url string, client http.Client, body []byte) PinotHTTP { +func NewHTTPClient(method, url string, client http.Client, body []byte, auth Auth) PinotHTTP { newClient := &Client{ Method: method, URL: url, HTTPClient: client, Body: body, + Auth: auth, } return newClient @@ -57,6 +67,10 @@ func (c *Client) Do() HttpResponse { return HttpResponse{Err: err} } + if c.Auth.BasicAuth != (BasicAuth{}) { + req.SetBasicAuth(c.Auth.BasicAuth.UserName, c.Auth.BasicAuth.Password) + } + req.Header.Add("Content-Type", "application/json") resp, err := c.HTTPClient.Do(req) if err != nil { diff --git a/internal/schema_controller/pinotschema_controller.go b/internal/schema_controller/pinotschema_controller.go index 74c34b2..541e913 100644 --- a/internal/schema_controller/pinotschema_controller.go +++ b/internal/schema_controller/pinotschema_controller.go @@ -58,6 +58,7 @@ func NewPinotSchemaReconciler(mgr ctrl.Manager) *PinotSchemaReconciler { // +kubebuilder:rbac:groups=datainfra.io,resources=pinotschemas,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=datainfra.io,resources=pinotschemas/status,verbs=get;update;patch // +kubebuilder:rbac:groups=datainfra.io,resources=pinotschemas/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=secret,verbs=get func (r *PinotSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logr := log.FromContext(ctx) diff --git a/internal/schema_controller/reconciler.go b/internal/schema_controller/reconciler.go index 97edf39..1de2f4c 100644 --- a/internal/schema_controller/reconciler.go +++ b/internal/schema_controller/reconciler.go @@ -46,7 +46,15 @@ const ( PinotSchemaControllerPatchStatusSuccess = "PinotSchemaControllerPatchStatusSuccess" PinotSchemaControllerPatchStatusFail = "PinotSchemaControllerPatchStatusFail" PinotSchemaControllerFinalizer = "pinotschema.datainfra.io/finalizer" - PinotControllerPort = "9000" +) + +const ( + ControlPlaneUserName = "CONTROL_PLANE_USERNAME" + ControlPlanePassword = "CONTROL_PLANE_PASSWORD" +) + +const ( + PinotControllerPort = "9000" ) func (r *PinotSchemaReconciler) do(ctx context.Context, schema *v1beta1.PinotSchema) error { @@ -55,15 +63,21 @@ func (r *PinotSchemaReconciler) do(ctx context.Context, schema *v1beta1.PinotSch builder.ToNewBuilderRecorder(builder.BuilderRecorder{Recorder: r.Recorder, ControllerName: "PinotSchemaController"}), ) + basicAuth, err := r.getAuthCreds(ctx, schema) + if err != nil { + return err + } + svcName, err := r.getControllerSvcUrl(schema.Namespace, schema.Spec.PinotCluster) if err != nil { return err } - _, err = r.CreateOrUpdate(schema, svcName, *build) + _, err = r.CreateOrUpdate(schema, svcName, *build, internalHTTP.Auth{BasicAuth: basicAuth}) if err != nil { return err } + if schema.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent @@ -86,7 +100,13 @@ func (r *PinotSchemaReconciler) do(ctx context.Context, schema *v1beta1.PinotSch if err != nil { return err } - http := internalHTTP.NewHTTPClient(http.MethodDelete, makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), http.Client{}, []byte{}) + http := internalHTTP.NewHTTPClient( + http.MethodDelete, + makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), + http.Client{}, + []byte{}, + internalHTTP.Auth{BasicAuth: basicAuth}, + ) resp := http.Do() if resp.Err != nil { build.Recorder.GenericEvent(schema, v1.EventTypeWarning, fmt.Sprintf("Resp [%s]", string(resp.RespBody)), PinotSchemaControllerDeleteFail) @@ -113,6 +133,7 @@ func (r *PinotSchemaReconciler) CreateOrUpdate( schema *v1beta1.PinotSchema, svcName string, build builder.Builder, + auth internalHTTP.Auth, ) (controllerutil.OperationResult, error) { // get schema name @@ -122,10 +143,16 @@ func (r *PinotSchemaReconciler) CreateOrUpdate( } // get schema - getHttp := internalHTTP.NewHTTPClient(http.MethodGet, makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), http.Client{}, []byte{}) + getHttp := internalHTTP.NewHTTPClient( + http.MethodGet, + makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), + http.Client{}, + []byte{}, + auth, + ) resp := getHttp.Do() if resp.Err != nil { - return controllerutil.OperationResultNone, err + return controllerutil.OperationResultNone, resp.Err } // if not found create schema @@ -133,7 +160,13 @@ func (r *PinotSchemaReconciler) CreateOrUpdate( if resp.StatusCode == 404 { // create schema - postHttp := internalHTTP.NewHTTPClient(http.MethodPost, makeControllerCreateSchemaPath(svcName), http.Client{}, []byte(schema.Spec.PinotSchemaJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPost, + makeControllerCreateSchemaPath(svcName), + http.Client{}, + []byte(schema.Spec.PinotSchemaJson), + auth, + ) respS := postHttp.Do() if respS.Err != nil { return controllerutil.OperationResultNone, err @@ -165,7 +198,13 @@ func (r *PinotSchemaReconciler) CreateOrUpdate( // if desiredstate and currentstate not the same then update if !ok { - postHttp := internalHTTP.NewHTTPClient(http.MethodPut, makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), http.Client{}, []byte(schema.Spec.PinotSchemaJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPut, + makeControllerGetUpdateDeleteSchemaPath(svcName, schemaName), + http.Client{}, + []byte(schema.Spec.PinotSchemaJson), + auth, + ) resp := postHttp.Do() if resp.Err != nil { return controllerutil.OperationResultNone, err @@ -267,3 +306,37 @@ func (r *PinotSchemaReconciler) getControllerSvcUrl(namespace, pinotClusterName return newName, nil } + +func (r *PinotSchemaReconciler) getAuthCreds(ctx context.Context, schema *v1beta1.PinotSchema) (internalHTTP.BasicAuth, error) { + pinot := v1beta1.Pinot{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: schema.Namespace, + Name: schema.Spec.PinotCluster, + }, + &pinot, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + if pinot.Spec.Auth != (v1beta1.Auth{}) { + secret := v1.Secret{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: pinot.Spec.Auth.SecretRef.Namespace, + Name: pinot.Spec.Auth.SecretRef.Name, + }, + &secret, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + creds := internalHTTP.BasicAuth{ + UserName: string(secret.Data[ControlPlaneUserName]), + Password: string(secret.Data[ControlPlanePassword]), + } + + return creds, nil + + } + + return internalHTTP.BasicAuth{}, nil +} diff --git a/internal/table_controller/pinottable_controller.go b/internal/table_controller/pinottable_controller.go index 3ecd1ec..e717296 100644 --- a/internal/table_controller/pinottable_controller.go +++ b/internal/table_controller/pinottable_controller.go @@ -55,10 +55,10 @@ func NewPinotTableReconciler(mgr ctrl.Manager) *PinotTableReconciler { } } -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottables,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottables/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottables/finalizers,verbs=update - +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottables,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottables/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottables/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=secret,verbs=get func (r *PinotTableReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logr := log.FromContext(ctx) diff --git a/internal/table_controller/reconciler.go b/internal/table_controller/reconciler.go index b25cc5b..8557dc5 100644 --- a/internal/table_controller/reconciler.go +++ b/internal/table_controller/reconciler.go @@ -47,7 +47,15 @@ const ( PinotTableControllerDeleteSuccess = "PinotTableControllerDeleteSuccess" PinotTableControllerDeleteFail = "PinotTableControllerDeleteFail" PinotTableControllerFinalizer = "pinottable.datainfra.io/finalizer" - PinotControllerPort = "9000" +) + +const ( + PinotControllerPort = "9000" +) + +const ( + ControlPlaneUserName = "CONTROL_PLANE_USERNAME" + ControlPlanePassword = "CONTROL_PLANE_PASSWORD" ) func (r *PinotTableReconciler) do(ctx context.Context, table *v1beta1.PinotTable) error { @@ -61,7 +69,11 @@ func (r *PinotTableReconciler) do(ctx context.Context, table *v1beta1.PinotTable return err } - _, err = r.CreateOrUpdate(table, svcName, *build) + basicAuth, err := r.getAuthCreds(ctx, table) + if err != nil { + return err + } + _, err = r.CreateOrUpdate(table, svcName, *build, internalHTTP.Auth{BasicAuth: basicAuth}) if err != nil { return err } @@ -86,7 +98,12 @@ func (r *PinotTableReconciler) do(ctx context.Context, table *v1beta1.PinotTable if err != nil { return err } - http := internalHTTP.NewHTTPClient(http.MethodDelete, makeControllerGetUpdateDeleteTablePath(svcName, tenantName), http.Client{}, []byte{}) + http := internalHTTP.NewHTTPClient( + http.MethodDelete, + makeControllerGetUpdateDeleteTablePath(svcName, tenantName), + http.Client{}, []byte{}, + internalHTTP.Auth{BasicAuth: basicAuth}, + ) resp := http.Do() if resp.Err != nil { build.Recorder.GenericEvent(table, v1.EventTypeWarning, fmt.Sprintf("Resp [%s]", string(resp.RespBody)), PinotTableControllerDeleteFail) @@ -112,6 +129,7 @@ func (r *PinotTableReconciler) CreateOrUpdate( table *v1beta1.PinotTable, svcName string, build builder.Builder, + auth internalHTTP.Auth, ) (controllerutil.OperationResult, error) { // get table name @@ -121,7 +139,12 @@ func (r *PinotTableReconciler) CreateOrUpdate( } // get table - getHttp := internalHTTP.NewHTTPClient(http.MethodGet, makeControllerGetUpdateDeleteTablePath(svcName, tableName), http.Client{}, []byte{}) + getHttp := internalHTTP.NewHTTPClient( + http.MethodGet, + makeControllerGetUpdateDeleteTablePath(svcName, tableName), + http.Client{}, []byte{}, + auth, + ) resp := getHttp.Do() if resp.Err != nil { return controllerutil.OperationResultNone, err @@ -130,7 +153,13 @@ func (r *PinotTableReconciler) CreateOrUpdate( // if not found create table if string(resp.RespBody) == "{}" { - postHttp := internalHTTP.NewHTTPClient(http.MethodPost, makeControllerCreateTablePath(svcName), http.Client{}, []byte(table.Spec.PinotTablesJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPost, + makeControllerCreateTablePath(svcName), + http.Client{}, + []byte(table.Spec.PinotTablesJson), + auth, + ) respT := postHttp.Do() if respT.Err != nil { return controllerutil.OperationResultNone, err @@ -159,7 +188,13 @@ func (r *PinotTableReconciler) CreateOrUpdate( } if !ok { - postHttp := internalHTTP.NewHTTPClient(http.MethodPut, makeControllerGetUpdateDeleteTablePath(svcName, tableName), http.Client{}, []byte(table.Spec.PinotTablesJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPut, + makeControllerGetUpdateDeleteTablePath(svcName, tableName), + http.Client{}, + []byte(table.Spec.PinotTablesJson), + auth, + ) resp := postHttp.Do() if resp.Err != nil { build.Recorder.GenericEvent(table, v1.EventTypeWarning, fmt.Sprintf("Resp [%s]", string(resp.RespBody)), PinotTableControllerCreateFail) @@ -256,3 +291,37 @@ func (r *PinotTableReconciler) makePatchPinotTableStatus( return controllerutil.OperationResultUpdatedStatusOnly, nil } + +func (r *PinotTableReconciler) getAuthCreds(ctx context.Context, table *v1beta1.PinotTable) (internalHTTP.BasicAuth, error) { + pinot := v1beta1.Pinot{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: table.Namespace, + Name: table.Spec.PinotCluster, + }, + &pinot, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + if pinot.Spec.Auth != (v1beta1.Auth{}) { + secret := v1.Secret{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: pinot.Spec.Auth.SecretRef.Namespace, + Name: pinot.Spec.Auth.SecretRef.Name, + }, + &secret, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + creds := internalHTTP.BasicAuth{ + UserName: string(secret.Data[ControlPlaneUserName]), + Password: string(secret.Data[ControlPlanePassword]), + } + + return creds, nil + + } + + return internalHTTP.BasicAuth{}, nil +} diff --git a/internal/tenant_controller/pinottenant_controller.go b/internal/tenant_controller/pinottenant_controller.go index fb513f1..8a5c477 100644 --- a/internal/tenant_controller/pinottenant_controller.go +++ b/internal/tenant_controller/pinottenant_controller.go @@ -55,10 +55,10 @@ func NewPinotTenantReconciler(mgr ctrl.Manager) *PinotTenantReconciler { } } -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottenants,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottenants/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=datainfra.io,resources=pinottenants/finalizers,verbs=update - +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottenants,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottenants/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=datainfra.io,resources=pinottenants/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=secret,verbs=get func (r *PinotTenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logr := log.FromContext(ctx) diff --git a/internal/tenant_controller/reconciler.go b/internal/tenant_controller/reconciler.go index 8b74316..9ffd23f 100644 --- a/internal/tenant_controller/reconciler.go +++ b/internal/tenant_controller/reconciler.go @@ -46,7 +46,15 @@ const ( PinotTenantControllerPatchStatusSuccess = "PinotTenantControllerPatchStatusSuccess" PinotTenantControllerPatchStatusFail = "PinotTenantControllerPatchStatusFail" PinotTenantControllerFinalizer = "pinottenant.datainfra.io/finalizer" - PinotControllerPort = "9000" +) + +const ( + PinotControllerPort = "9000" +) + +const ( + ControlPlaneUserName = "CONTROL_PLANE_USERNAME" + ControlPlanePassword = "CONTROL_PLANE_PASSWORD" ) func (r *PinotTenantReconciler) do(ctx context.Context, tenant *v1beta1.PinotTenant) error { @@ -59,7 +67,12 @@ func (r *PinotTenantReconciler) do(ctx context.Context, tenant *v1beta1.PinotTen return err } - _, err = r.CreateOrUpdate(tenant, svcName, *build) + basicAuth, err := r.getAuthCreds(ctx, tenant) + if err != nil { + return err + } + + _, err = r.CreateOrUpdate(tenant, svcName, *build, internalHTTP.Auth{BasicAuth: basicAuth}) if err != nil { return err } @@ -87,7 +100,14 @@ func (r *PinotTenantReconciler) do(ctx context.Context, tenant *v1beta1.PinotTen if err != nil { return err } - http := internalHTTP.NewHTTPClient(http.MethodDelete, makeControllerDeleteTenantPath(svcName, tenantName, string(tenant.Spec.PinotTenantType)), http.Client{}, []byte{}) + http := internalHTTP.NewHTTPClient( + http.MethodDelete, + makeControllerDeleteTenantPath(svcName, tenantName, + string(tenant.Spec.PinotTenantType)), + http.Client{}, + []byte{}, + internalHTTP.Auth{BasicAuth: basicAuth}, + ) resp := http.Do() if resp.Err != nil { build.Recorder.GenericEvent(tenant, v1.EventTypeWarning, fmt.Sprintf("Resp [%s]", string(resp.RespBody)), PinotTenantControllerDeleteFail) @@ -168,6 +188,7 @@ func (r *PinotTenantReconciler) CreateOrUpdate( tenant *v1beta1.PinotTenant, svcName string, build builder.Builder, + auth internalHTTP.Auth, ) (controllerutil.OperationResult, error) { // get tenant name @@ -177,7 +198,13 @@ func (r *PinotTenantReconciler) CreateOrUpdate( } // get tenant - getHttp := internalHTTP.NewHTTPClient(http.MethodGet, makeControllerGetTenantPath(svcName, tenantName), http.Client{}, []byte{}) + getHttp := internalHTTP.NewHTTPClient( + http.MethodGet, + makeControllerGetTenantPath(svcName, tenantName), + http.Client{}, + []byte{}, + auth, + ) resp := getHttp.Do() if resp.Err != nil { build.Recorder.GenericEvent(tenant, v1.EventTypeWarning, fmt.Sprintf("Resp [%s]", string(resp.RespBody)), PinotTenantControllerGetFail) @@ -187,7 +214,13 @@ func (r *PinotTenantReconciler) CreateOrUpdate( // if not found create tenant if resp.StatusCode == 404 { - postHttp := internalHTTP.NewHTTPClient(http.MethodPost, makeControllerCreateUpdateTenantPath(svcName), http.Client{}, []byte(tenant.Spec.PinotTenantsJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPost, + makeControllerCreateUpdateTenantPath(svcName), + http.Client{}, + []byte(tenant.Spec.PinotTenantsJson), + auth, + ) respT := postHttp.Do() if respT.Err != nil { return controllerutil.OperationResultNone, respT.Err @@ -215,7 +248,13 @@ func (r *PinotTenantReconciler) CreateOrUpdate( return controllerutil.OperationResultNone, err } if !ok { - postHttp := internalHTTP.NewHTTPClient(http.MethodPut, makeControllerCreateUpdateTenantPath(svcName), http.Client{}, []byte(tenant.Spec.PinotTenantsJson)) + postHttp := internalHTTP.NewHTTPClient( + http.MethodPut, + makeControllerCreateUpdateTenantPath(svcName), + http.Client{}, + []byte(tenant.Spec.PinotTenantsJson), + auth, + ) respUpdate := postHttp.Do() if respUpdate.Err != nil { return controllerutil.OperationResultNone, err @@ -274,3 +313,37 @@ func (r *PinotTenantReconciler) makePatchPinotTenantStatus( return controllerutil.OperationResultUpdatedStatusOnly, nil } + +func (r *PinotTenantReconciler) getAuthCreds(ctx context.Context, tenant *v1beta1.PinotTenant) (internalHTTP.BasicAuth, error) { + pinot := v1beta1.Pinot{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: tenant.Namespace, + Name: tenant.Spec.PinotCluster, + }, + &pinot, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + if pinot.Spec.Auth != (v1beta1.Auth{}) { + secret := v1.Secret{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: pinot.Spec.Auth.SecretRef.Namespace, + Name: pinot.Spec.Auth.SecretRef.Name, + }, + &secret, + ); err != nil { + return internalHTTP.BasicAuth{}, err + } + + creds := internalHTTP.BasicAuth{ + UserName: string(secret.Data[ControlPlaneUserName]), + Password: string(secret.Data[ControlPlanePassword]), + } + + return creds, nil + + } + + return internalHTTP.BasicAuth{}, nil +} From eefafabcb6c5bab323d28c92b66630e460fd2eb7 Mon Sep 17 00:00:00 2001 From: AdheipSingh Date: Sat, 29 Apr 2023 16:55:15 +0530 Subject: [PATCH 2/2] update readme --- docs/getting_started_auth.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting_started_auth.md b/docs/getting_started_auth.md index f3ccefd..a29b350 100644 --- a/docs/getting_started_auth.md +++ b/docs/getting_started_auth.md @@ -26,7 +26,8 @@ envsubst < examples/04-pinot-auth/pinotauth-basic.yaml | kubectl apply -f - -n ### Create a K8 secret in the namespace where pinot cluster is deployed -- add secrets to file +- add secrets to file, the following secrets needs to be same as mentioned + in pinot controller properties. ``` cat << EOF > pinot-control-plane-secret CONTROL_PLANE_USERNAME=controlplane