Merge pull request #29 from datainfrahq/aws

add aws examples
datainfrahq · Apr 29, 2023 · d32c854 · d32c854
2 parents 496a6b3 + 59e043d
commit d32c854
Show file tree

Hide file tree

Showing 4 changed files with 749 additions and 0 deletions.
diff --git a/docs/getting_started_aws.md b/docs/getting_started_aws.md
@@ -0,0 +1,63 @@
+#### Export your StorageClassName 
+```
+export STORAGE_CLASS_NAME=gp2
+```
+
+#### Install Pinot Control Plane
+```
+make helm-install-pinot-control-plane
+```
+
+#### Install Zookeeper Opoerator and CR
+```
+make helm-install-zk-operator
+```
+#### Install Pinot Cluster
+```
+envsubst < examples/05-pinot-aws/pinot-aws.yaml  | kubectl apply -f - -n pinot
+```
+#### Deploy Kafka Cluster and Create Topics
+```
+# Add Kafka
+helm repo add kafka https://charts.bitnami.com/bitnami
+# Deploy kafka
+helm install -n pinot kafka kafka/kafka --set replicas=1,zookeeper.image.tag=latest
+# Create topics
+kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime --create --partitions 1 --replication-factor 1
+kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime-avro --create --partitions 1 --replication-factor 1
+```
+
+#### Access Pinot Console
+
+```
+kubectl port-forward svc/pinot-controller-controller-svc -n pinot 9000
+```
+
+### Once Pinot Cluster is up and running
+
+#### Create Schema
+```
+kubectl apply -f examples/05-pinot-aws/pinotschema-aws.yaml -n pinot
+```
+
+#### Create Table
+```
+kubectl apply -f examples/05-pinot-aws/pinottable-aws.yaml -n pinot
+```
+
+#### Check All Custom Resources created by the control plane
+```
+kubectl get pinot -A
+kubectl get pinotschema -A
+kubectl get pinottable -A
+```
+
+#### Load Data Into Kafka
+```
+kubectl apply -f examples/03-pinot-minio/pinot-realtime-kafka.yaml
+```
+
+#### Port-forward and query on console
+```
+kubectl port-forward pinot-controller-controller-0 -n pinot 9000
+```
diff --git a/examples/05-pinot-aws/pinotbasic-aws.yaml b/examples/05-pinot-aws/pinotbasic-aws.yaml
@@ -0,0 +1,286 @@
+apiVersion: datainfra.io/v1beta1
+kind: Pinot
+metadata:
+  name: pinot-aws
+spec:
+
+  plugins:
+   - pinot-s3
+
+  external:
+
+    zookeeper:
+     spec:
+       zkAddress: zk-pinot-zookeeper-headless.pinot:2181
+
+    deepStorage:
+     spec:
+      - nodeType: controller
+        data: |-
+          controller.data.dir=s3://pinot-datainfra/pinot-realtime
+          pinot.controller.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
+          pinot.controller.storage.factory.s3.region=us-east-1
+          pinot.controller.storage.factory.s3.accessKey=
+          pinot.controller.storage.factory.s3.secretKey=
+          pinot.controller.segment.fetcher.protocols=file,http,s3
+          pinot.controller.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
+      - nodeType: server
+        data: |-
+          pinot.server.instance.enable.split.commit=true
+          pinot.server.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
+          pinot.server.storage.factory.s3.region=us-west-2
+          pinot.server.storage.factory.s3.httpclient.maxConnections=50
+          pinot.server.storage.factory.s3.httpclient.socketTimeout=30s
+          pinot.server.storage.factory.s3.httpclient.connectionTimeout=2s
+          pinot.server.storage.factory.s3.httpclient.connectionTimeToLive=0s
+          pinot.server.storage.factory.s3.httpclient.connectionAcquisitionTimeout=10s
+          pinot.server.segment.fetcher.protocols=file,http,s3
+          pinot.server.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
+      - nodeType: minion
+        data: |-
+          pinot.minion.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
+          pinot.minion.storage.factory.s3.region=us-west-2
+          pinot.minion.segment.fetcher.protocols=file,http,s3
+          pinot.minion.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
+
+  nodes:
+
+    - name: pinot-controller
+      kind: Statefulset
+      replicas: 1
+      nodeType: controller
+      k8sConfig: controller
+      pinotNodeConfig: controller
+
+    - name: pinot-broker
+      kind: Statefulset
+      replicas: 1
+      nodeType: broker
+      k8sConfig: broker
+      pinotNodeConfig: broker
+
+    - name: pinot-server
+      kind: Statefulset
+      replicas: 1
+      nodeType: server
+      k8sConfig: server
+      pinotNodeConfig: server
+
+    - name: pinot-minion
+      kind: Statefulset
+      replicas: 1
+      nodeType: minion
+      k8sConfig: minion
+      pinotNodeConfig: minion
+
+  deploymentOrder:
+  - controller
+  - broker
+  - server
+  - minion
+
+  k8sConfig:
+
+  - name: controller
+    serviceAccountName: "default"
+    port:
+    - name: controller 
+      containerPort: 9000
+      protocol: TCP
+    livenessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 9000
+    readinessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 9000
+    service:
+      type: ClusterIP
+      ports:
+      - protocol: TCP
+        port: 9000
+        targetPort: 9000
+    env:
+    - name: LOG4J_CONSOLE_LEVEL
+      value: debug
+    image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
+    storageConfig:
+    - name: pinotcontroller
+      mountPath: "/var/pinot/controller/data"
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: ${STORAGE_CLASS_NAME}
+        resources:
+          requests:
+            storage: 10Gi
+
+  - name: broker
+    serviceAccountName: "default"
+    port:
+    - name: broker 
+      containerPort: 8099
+      protocol: TCP
+    livenessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 8099
+    readinessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 8099
+    service:
+      type: ClusterIP
+      ports:
+      - name: broker
+        port: 8099
+        protocol: TCP
+        targetPort: 8099
+    env:
+    - name: LOG4J_CONSOLE_LEVEL
+      value: info
+    image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
+    storageConfig:
+    - name: broker
+      mountPath: "/var/pinot/broker/data"
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: ${STORAGE_CLASS_NAME}
+        resources:
+          requests:
+            storage: 10Gi
+
+  - name: server
+    serviceAccountName: "default"
+    port:
+    - name: server 
+      containerPort: 8097
+      protocol: TCP
+    livenessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 8097
+    readinessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 8097
+    service:
+      type: ClusterIP
+      ports:
+      - name: netty
+        port: 8098
+        protocol: TCP
+        targetPort: 8098
+      - protocol: TCP
+        port: 80
+        targetPort: 8097
+        name: admin
+    image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
+    env:
+    - name: LOG4J_CONSOLE_LEVEL
+      value: info
+    storageConfig:
+    - name: server
+      mountPath: "/var/pinot/server/data"
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: ${STORAGE_CLASS_NAME}
+        resources:
+          requests:
+            storage: 10Gi
+
+  - name: minion
+    serviceAccountName: "default"
+    image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
+    livenessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 9514
+    readinessProbe:
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      httpGet:
+        path: "/health"
+        port: 9514
+    service:
+      type: ClusterIP
+      ports:
+      - name: minion
+        port: 9514
+        protocol: TCP
+        targetPort: 9514
+    port:
+    - name: minion 
+      containerPort: 9514
+      protocol: TCP
+    env:
+    - name: LOG4J_CONSOLE_LEVEL
+      value: info
+    storageConfig:
+    - name: minion
+      mountPath: "/var/pinot/minion/data"
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: ${STORAGE_CLASS_NAME}
+        resources:
+          requests:
+            storage: 10Gi 
+
+  pinotNodeConfig:
+
+  - name: controller
+    java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
+                -Xlog:gc*:file=/opt/pinot/gc-pinot-controller.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
+                -Dplugins.dir=/opt/pinot/plugins"
+    data: |-
+        controller.port=9000
+        controller.local.temp.dir=/var/pinot/controller/data
+        pinot.set.instance.id.to.hostname=true
+        controller.task.scheduler.enabled=true
+
+  - name: broker
+    java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
+                -Xlog:gc*:file=/opt/pinot/gc-pinot-broker.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
+                -Dplugins.dir=/opt/pinot/plugins"
+    data: |-
+      pinot.broker.client.queryPort=8099
+      pinot.broker.routing.table.builder.class=random
+      pinot.set.instance.id.to.hostname=true
+
+  - name: server
+    java_opts: "-Xms512M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-server.log
+                -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml -Dplugins.dir=/opt/pinot/plugins"
+    data: |-
+      pinot.server.netty.port=8098
+      pinot.server.adminapi.port=8097
+      pinot.server.instance.dataDir=/var/pinot/server/data/index
+      pinot.server.instance.segmentTarDir=/var/pinot/server/data/segment
+      pinot.set.instance.id.to.hostname=true
+      pinot.server.instance.realtime.alloc.offheap=true
+  
+  - name: minion
+    java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
+                -Xlog:gc*:file=/opt/pinot/gc-pinot-minion.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
+                -Dplugins.dir=/opt/pinot/plugins"
+    data: |-
+      pinot.minion.port=9514
+      dataDir=/var/pinot/minion/data
+      pinot.set.instance.id.to.hostname=true