Merge pull request #22 from datainfrahq/tables-crd

Support For Table Operations

Makefile

@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= datainfrahq/pinot-control-plane:v0.0.3
+IMG ?= datainfrahq/pinot-control-plane:v0.0.4
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.26.0
 

PROJECT

@@ -5,8 +5,8 @@
 domain: datainfra.io
 layout:
 - go.kubebuilder.io/v4-alpha
-projectName: pinot-operator
-repo: github.com/datainfrahq/pinot-operator
+projectName: pinot-control-plane-k8s
+repo: github.com/datainfrahq/pinot-control-plane-k8s
 resources:
 - api:
     crdVersion: v1
@@ -15,7 +15,7 @@ resources:
   domain: datainfra.io
   group: datainfra.io
   kind: Pinot
-  path: github.com/datainfrahq/pinot-operator/api/v1beta1
+  path: github.com/datainfrahq/pinot-control-plane-k8s/api/v1beta1
   version: v1beta1
 - api:
     crdVersion: v1
@@ -24,6 +24,15 @@ resources:
   domain: datainfra.io
   group: datainfra.io
   kind: PinotSchema
-  path: github.com/datainfrahq/pinot-operator/api/v1beta1
+  path: github.com/datainfrahq/pinot-control-plane-k8s/api/v1beta1
+  version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: datainfra.io
+  group: datainfra.io
+  kind: PinotTable
+  path: github.com/datainfrahq/pinot-control-plane-k8s/api/v1beta1
   version: v1beta1
 version: "3"

README.md

@@ -22,12 +22,66 @@ Control Plane for deploying and managing heterogenous apache pinot kubernetes cl
 
 ### Getting Started 
 
+- Export your StorageClassName 
+```
+export STORAGE_CLASS_NAME=standard
+```
+
+- Install Pinot Control Plane
 ```
-export STORAGE_CLASS_NAME=civo-volume
 make helm-install-pinot-control-plane
+```
+
+- Install Zookeeper Opoerator and CR
+```
 make helm-install-zk-operator
+```
+
+- Install Pinot Cluster
+```
 envsubst < examples/pinot-simple.yaml  | kubectl apply -f - -n pinot
 ```
+
+- Deploy Kafka Cluster and Create Topics
+```
+# Add Kafka
+helm repo add kafka https://charts.bitnami.com/bitnami
+# Deploy kafka
+helm install -n pinot kafka kafka/kafka --set replicas=1,zookeeper.image.tag=latest
+# Create topics
+kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime --create --partitions 1 --replication-factor 1
+kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime-avro --create --partitions 1 --replication-factor 1
+```
+
+- Create Schema
+```
+kubectl apply -f examples/pinotschema-simple.yaml -n pinot
+```
+
+- Create Table
+```
+kubectl apply -f examples/pinottable-simple.yaml -n pinot
+```
+
+- Check All Custom Resources created by the control plane
+```
+kubectl get pinot -A
+kubectl get pinotSchema -A
+kubectl get pinottable -A
+```
+
+- Load Data Into Kafka
+```
+kubectl apply -f examples/pinot/pinot-realtime-kafka.yaml
+```
+
+- Port-forward and query on console
+```
+```
+kubectl port-forward pinot-controller-controller-0 -n pinot 9000
+```
+```
+
 ### Getting Started With DeepStorage Minio
 
 - An e2e getting started from kafka > pinot > minio s3.

api/v1beta1/pinotschema_types.go

@@ -22,8 +22,10 @@ import (
 
 // PinotSchemaSpec defines the desired state of PinotSchema
 type PinotSchemaSpec struct {
-	ClusterName string `json:"clusterName"`
-	SchemaJson  string `json:"schema.json,omitempty"`
+	// +required
+	PinotCluster string `json:"pinotCluster"`
+	// +required
+	PinotSchemaJson string `json:"schema.json"`
 }
 
 // PinotSchemaStatus defines the observed state of PinotSchema
@@ -33,7 +35,7 @@ type PinotSchemaStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
-// +kubebuilder:printcolumn:name="Pinot_Cluster",type="string",JSONPath=".spec.clusterName"
+// +kubebuilder:printcolumn:name="Pinot_Cluster",type="string",JSONPath=".spec.pinotCluster"
 // PinotSchema is the Schema for the pinotschemas API
 type PinotSchema struct {
 	metav1.TypeMeta   `json:",inline"`

api/v1beta1/pinottable_types.go

@@ -0,0 +1,71 @@
+/*
+DataInfra Pinot Control Plane (C) 2023 - 2024 DataInfra.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type PinotTableType string
+
+const (
+	RealTimeTable    PinotTableType = "realtime"
+	OfflineTimeTable PinotTableType = "offline"
+)
+
+// PinotTableSpec defines the desired state of PinotTable
+type PinotTableSpec struct {
+	// +required
+	PinotCluster string `json:"pinotCluster"`
+	// +required
+	PinotSchema string `json:"pinotSchema"`
+	// +required
+	PinotTableType PinotTableType `json:"pinotTableType"`
+	// +required
+	PinotTablesJson string `json:"tables.json"`
+}
+
+// PinotTableStatus defines the observed state of PinotTable
+type PinotTableStatus struct {
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Pinot_Cluster",type="string",JSONPath=".spec.pinotCluster"
+// +kubebuilder:printcolumn:name="Pinot_Schema",type="string",JSONPath=".spec.pinotSchema"
+// PinotTable is the Schema for the pinottables API
+type PinotTable struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PinotTableSpec   `json:"spec,omitempty"`
+	Status PinotTableStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// PinotTableList contains a list of PinotTable
+type PinotTableList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PinotTable `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PinotTable{}, &PinotTableList{})
+}

cmd/main.go

@@ -34,6 +34,7 @@ import (
 	datainfraiov1beta1 "github.com/datainfrahq/pinot-control-plane-k8s/api/v1beta1"
 	pinotcontroller "github.com/datainfrahq/pinot-control-plane-k8s/internal/pinot_controller"
 	schemacontroller "github.com/datainfrahq/pinot-control-plane-k8s/internal/schema_controller"
+	tablecontroller "github.com/datainfrahq/pinot-control-plane-k8s/internal/table_controller"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -94,10 +95,15 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "PinotController")
 		os.Exit(1)
 	}
-	if err = (schemacontroller.NewPinotReconciler(mgr)).SetupWithManager(mgr); err != nil {
+	if err = (schemacontroller.NewPinotSchemaReconciler(mgr)).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "PinotSchemaController")
 		os.Exit(1)
 	}
+	if err = (tablecontroller.NewPinotTableReconciler(mgr)).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "PinotTableController")
+		os.Exit(1)
+	}
+
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/crd/bases/datainfra.io_pinotschemas.yaml

@@ -19,7 +19,7 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
-    - jsonPath: .spec.clusterName
+    - jsonPath: .spec.pinotCluster
       name: Pinot_Cluster
       type: string
     name: v1beta1
@@ -42,12 +42,13 @@ spec:
           spec:
             description: PinotSchemaSpec defines the desired state of PinotSchema
             properties:
-              clusterName:
+              pinotCluster:
                 type: string
               schema.json:
                 type: string
             required:
-            - clusterName
+            - pinotCluster
+            - schema.json
             type: object
           status:
             description: PinotSchemaStatus defines the observed state of PinotSchema