Merge branch 'release/1.3.0'

* release/1.3.0: (119 commits)
  release 1.3.0 (#282)
  check version only on develop (#281)
  Enable FluentD Prometheus Endpoint (#257)
  trigger functional test from sub plugin repos (#279)
  add latest hec image to firelens readme (#280)
  add latest hec image to firelens readme (#280)
  Firelens (#260)
  ingest api support (#253)
  abstract HEC info (#254)
  For multiline events, use the timestamp of the first event and d… (#244)
  remove version label from selector in manifests (#229)
  added indexed fields to logging and objects (#231)
  Missing end clause (#238)
  Add support for kube apiserver audit logs (#232)
  updated configuration for systemd plugin to use matches as filte… (#233)
  index-routing-objects (#235)
  openshift fix (#226)
  Docker private images (#222)
  Update README to reflect support policy (#227)
  Custom filters (#216)
  ...

Author: rockb1017

.circleci/build_sck_artifacts.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
-TAG="1.1.0.Alpha-${CIRCLE_SHA1}"
+VERSION=`cat VERSION`
+TAG="$VERSION-${CIRCLE_SHA1}"
 
 function replace_generic_version ()
 {

.circleci/build_sck_artifacts_for_release.sh

@@ -1,16 +1,20 @@
 #!/usr/bin/env bash
 set -e
-TAG=`cat VERSION`
+VERSION=`cat VERSION`
+
+source PLUGIN_VERSIONS.sh
 
 # Install yq yaml parser
 wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64
 sudo chmod +x yq_linux_amd64
 sudo mv yq_linux_amd64 /usr/local/bin/yq
 
-yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name splunk/fluentd-hec:1.1.1
-yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name splunk/k8s-metrics:1.1.1
-yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name splunk/k8s-metrics-aggr:1.1.0
-yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name splunk/kube-objects:1.1.0
+# Before release, "make manifests" call should have been run. 
+# then there are nothing else to change. but keeping them here just in case 
+yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.tag $FLUENTD_HEC_VERSION
+yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.tag $K8S_METRICS_VERISION
+yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.tag $K8S_METRICS_AGGR_VERSION
+yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.tag $KUBE_OBJECT_VERSION
 
 yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION
 yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION
@@ -22,9 +26,9 @@ yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml appVersion $VERSION
 yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml appVersion $VERSION
 yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml appVersion $VERSION
 
-yq w -i helm-chart/splunk-kubernetes-logging/requirements.yaml dependencies.splunk-kubernetes-logging.version $VERSION
-yq w -i helm-chart/splunk-kubernetes-metrics/requirements.yaml dependencies.splunk-kubernetes-metrics.version $VERSION
-yq w -i helm-chart/splunk-kubernetes-objects/requirements.yaml dependencies.splunk-kubernetes-objects.version $VERSION
+yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[0].version $VERSION
+yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[1].version $VERSION
+yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[2].version $VERSION
 
 mkdir helm-artifacts-release
 if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then

.circleci/chart_schema.yaml

@@ -0,0 +1,20 @@
+name: str()
+home: str()
+version: str()
+appVersion: any(str(), num())
+description: str()
+keywords: list(str(), required=False)
+sources: list(str(), required=False)
+maintainers: list(include('maintainer'), required=False)
+icon: str(required=False)
+engine: str(required=False)
+condition: str(required=False)
+tags: str(required=False)
+deprecated: bool(required=False)
+kubeVersion: str(required=False)
+annotations: map(str(), str(), required=False)
+---
+maintainer:
+  name: str()
+  email: str(required=False)
+  url: str(required=False)

.circleci/check_version_trigger_release.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+LATEST_COMMIT=$(git rev-parse HEAD)
+VERSION_COMMIT=$(git log -1 --format=format:%H VERSION)
+if [ $VERSION_COMMIT = $LATEST_COMMIT ];
+    then
+        if [ -s VERSION ] # Check if content is empty
+            then 
+                VERSION=`cat VERSION`
+                echo "VERSION is changed to $VERSION" 
+            else 
+                echo "[ERROR] VERSION file is empty."
+                exit 1
+        fi 
+        git config user.email "splunk-oss-admin@splunk.com"
+        git config user.name "splunk-oss-admin"
+        git checkout develop
+        git pull origin develop
+        git checkout -b release/$VERSION origin/develop
+        git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/splunk-connect-for-kubernetes.git release/$VERSION
+        git checkout master
+        git merge --no-edit release/$VERSION
+        git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/splunk-connect-for-kubernetes.git master
+fi

.circleci/config.yml

@@ -1,13 +1,93 @@
 version: 2
 jobs:
-  build:
-    docker:
-      - image: circleci/ruby:2.6.1-node-browsers
+  build_test:
+    resource_class: large
+    machine:
+      image: ubuntu-1604:201903-01
+    environment:
+      CHANGE_MINIKUBE_NONE_USER: true
+      GOPATH: /home/circleci/go
+      KUBECONFIG: /home/circleci/.kube/config
+      KUBERNETES_VERSION: v1.15.2
+      MINIKUBE_HOME: /home/circleci
+      MINIKUBE_VERSION: v1.2.0
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+      SKAFFOLD_VERSION: v0.33.0
+      GO111MODULE: "on"
+      CI_SPLUNK_VERSION: "7.3.2"
+      CI_SPLUNK_FILENAME: splunk-7.3.2-c60db69f8e32-linux-2.6-amd64.deb
+      CI_SPLUNK_HOST: 127.0.0.1
+      CI_SPLUNK_PORT: 8089
+      CI_SPLUNK_USERNAME: admin
+      CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
+      CI_SPLUNK_HEC_TOKEN_PERF: a6b5e77f-d5f6-415a-bd43-930cecb12959
+      CI_SPLUNK_PASSWORD: helloworld
+      CI_HEC_PROTOCOL: http
+      CI_INDEX_EVENTS: circleci_events
+      CI_INDEX_OBJECTS: circleci_objects
+      CI_INDEX_METRICS: circleci_metrics
+      CI_INDEX_NAMESPACE_ROUTING: test-namespace-routing
+      CI_KAFKA_DATAGEN_IMAGE: chaitanyaphalak/kafkadatagen:1.0-4-gca7f6d8
     working_directory: ~/repo
     steps:
       - checkout
-      - setup_remote_docker:
-          reusable: true
+      - run:
+          name: Install Splunk
+          command: |
+            cd /tmp && wget -O $CI_SPLUNK_FILENAME 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version='$CI_SPLUNK_VERSION'&product=splunk&filename='$CI_SPLUNK_FILENAME'&wget=true'
+            sudo dpkg -i $CI_SPLUNK_FILENAME
+            # Set user seed
+            hashed_pwd=$(sudo /opt/splunk/bin/splunk hash-passwd $CI_SPLUNK_PASSWORD)
+            sudo tee /opt/splunk/etc/system/local/user-seed.conf > /dev/null << EOF
+            [user_info]
+            USERNAME = $CI_SPLUNK_USERNAME
+            HASHED_PASSWORD = $hashed_pwd
+            EOF
+            # Add delete capability to admin role
+            sudo tee -a /opt/splunk/etc/system/local/authorize.conf > /dev/null << EOF
+            [role_admin]
+            delete_by_keyword = enabled
+            EOF
+            # start Splunk
+            sudo /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt
+            # Enable HEC services
+            curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
+            # Create new HEC token
+            curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=$CI_SPLUNK_HEC_TOKEN" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
+            # Enable HEC new-token
+            sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            # Disable SSL for HEC new-token
+            sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -enable-ssl 0 -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            # Setup Indexes
+            sudo /opt/splunk/bin/splunk add index $CI_INDEX_EVENTS -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk add index $CI_INDEX_OBJECTS -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk add index $CI_INDEX_METRICS -datatype metric -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk add index kube-system -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk add index kube-public -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk add index $CI_INDEX_NAMESPACE_ROUTING -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
+            sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -name splunk_hec_token -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -index $CI_INDEX_EVENTS -indexes $CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,$CI_INDEX_NAMESPACE_ROUTING,kube-system,kube-public
+            # Restart Splunk
+            sudo /opt/splunk/bin/splunk restart --accept-license --answer-yes --no-prompt
+      - run:
+          name: Install and Start minikube
+          command: |
+            # Install Kubectl
+            curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
+            chmod +x kubectl
+            sudo mv kubectl /usr/local/bin/
+            mkdir -p ${HOME}/.kube
+            touch ${HOME}/.kube/config
+            # Install Minikube
+            curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+            chmod +x minikube
+            sudo mv minikube /usr/local/bin/  
+            # Start Minikube and Wait
+            sudo -E minikube start --vm-driver=none --cpus 2 --memory 4098 --kubernetes-version=${KUBERNETES_VERSION}
+            export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+            until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+              sleep 1;
+            done 
       - run:
           name: Install dependencies
           command: |
@@ -17,32 +97,36 @@ jobs:
             sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1
             curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3
             helm init -c
-            echo "Installing aws cli..."
-            sudo pip install awscli > /dev/null 2>&1
+            docker pull $CI_KAFKA_DATAGEN_IMAGE
+            pyenv global 3.6.5
+            pip install --upgrade pip
+            pip install -r test/requirements.txt
       - run:
-          name: Update configurables to build artifacts
+          name: Lint helm charts
           command: |
-            .circleci/update_sck_config.sh
+            .circleci/lint.sh
       - run:
           name: Build SCK artifacts
           command: |
             .circleci/build_sck_artifacts.sh
       - run:
-          name: Push SCK artifacts to s3
+          name: Update configurables for unit test
           command: |
-            .circleci/push_sck_artifacts.sh
+            export CI_SPLUNK_HEC_HOST=$(hostname -i)
+            export CI_SPLUNK_HEC_HOST_PERF=$(hostname -i)
+            .circleci/update_sck_config.sh
       - run:
           name: Deploy k8s connector
           command: |
             .circleci/deploy_connector.sh
       - run:
           name: Run functional tests
           command: |
+            export CI_SPLUNK_HEC_HOST=$(hostname -i)
+            pyenv global 3.6.5
+            export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+            sleep 30
             .circleci/run_functional_tests.sh
-      - run:
-          name:  Teardown deployment
-          command: |
-            helm delete --purge ci-$CIRCLE_SHA1
 
   perf-test:
     docker:
@@ -93,10 +177,6 @@ jobs:
             sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1
             curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
             helm init -c
-      - run:
-          name: Update configurables to build artifacts
-          command: |
-            .circleci/update_sck_config_for_release.sh
       - run:
           name: Build SCK artifacts
           command: |
@@ -105,32 +185,52 @@ jobs:
           name: Push SCK artifacts to Github Release
           command: |
             .circleci/push_sck_release.sh
+
+  check_version:
+    docker:
+      - image: circleci/ruby:2.6.1-node-browsers
+    working_directory: ~/repo
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - checkout          
+      - setup_remote_docker:
+          reusable: true
       - run:
-          name: Deploy k8s connector
-          command: |
-            .circleci/deploy_connector_release.sh
-      - run:
-          name: Run functional tests
+          name: Check VERSION file for change
           command: |
-            .circleci/run_functional_tests.sh
-
-# Need to add teardown step
+            .circleci/check_version_trigger_release.sh
 
 workflows:
   version: 2
-  build_perf-test:
+  build_test:
     jobs:
-      - build:
+      - build_test:
           filters:
             branches:
-              ignore: perf-testing
+              ignore: 
+                - perf-testing
+                - /^release\/.*/
+                - master
+      - check_version:
+          requires:
+            - build_test
+          filters:
+            branches:
+              only: develop
+  perf-test:
+    jobs:
       - perf-test:
           filters:
             branches:
               only: perf-testing
-      - release:
-          requires:
-          - build
+  release:
+    jobs:
+      - build_test:
           filters:
             branches:
-              only: release-testing
+              only: 
+                - master
+      - release:
+          requires:
+            - build_test

.circleci/deploy_connector.sh

@@ -1,15 +1,11 @@
 #!/usr/bin/env bash
 set -e
-echo "Setup kube client..."
-curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
-chmod +x ./kubectl
-sudo mv ./kubectl /usr/local/bin/kubectl
-sudo apt-get -y install gnupg
-sudo mkdir ~/.kube
-echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg
-sudo mv config ~/.kube/config
 #Update helm server version
-helm init --upgrade
+helm init --force-upgrade
+# Wait for helm to be ready 
+until kubectl get pod --all-namespaces | grep tiller | grep -q "1\/1"; do
+   sleep 1;
+done 
 #Make sure to check and clean previously failed deployment
 echo "Checking if previous deployment exist..."
 if [ "`helm ls`" == "" ]; then
@@ -19,6 +15,7 @@ else
 fi
 echo "Deploying k8s-connect with latest changes"
 helm install --name=ci-$CIRCLE_SHA1 -f .circleci/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz
-#Todo-need to add test cases and run them before cleaning the deployment
-#wait for deployment to finish, sleep longer for metrics-aggr to start sending metrics
-sleep 90
+#wait for deployment to finish
+until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do
+   sleep 1;
+done

.circleci/lint.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+# chart template linting
+wget https://github.com/helm/chart-testing/releases/download/v2.3.3/chart-testing_2.3.3_linux_amd64.tar.gz
+tar -xvf chart-testing_2.3.3_linux_amd64.tar.gz
+sudo chmod +x ct
+sudo mv ct /usr/local/bin/ct
+
+# lint helm charts
+echo "Linting charts"
+helm lint helm-chart/*
+echo "Linting charts with CI values"
+helm lint --values .circleci/sck_values.yml helm-chart/*
+cd .circleci
+echo "Linting charts ct lint tool"
+ct lint helm-chart/*