# This is a combination of 127 commits.

# This is the 1st commit message:

Revert "introduce arcmode"

This reverts commit 5f4fed4.

# This is the commit message #2:

remove secrets

# This is the commit message Azure#3:

add print statement

# This is the commit message Azure#4:

update print statement

# This is the commit message Azure#5:

committed

# This is the commit message Azure#6:

committed

# This is the commit message Azure#7:

committed

# This is the commit message Azure#8:

committed

# This is the commit message Azure#9:

remove print statements

# This is the commit message Azure#10:

add helm collector

# This is the commit message Azure#11:

change helm command

# This is the commit message Azure#12:

add helm 3 installation

# This is the commit message Azure#13:

add curl command installation

# This is the commit message Azure#14:

change helm command

# This is the commit message Azure#15:

remove helm history

# This is the commit message Azure#16:

debug helm history

# This is the commit message Azure#17:

add repo command

# This is the commit message Azure#18:

change stable repo name

# This is the commit message Azure#19:

add write to file

# This is the commit message Azure#20:

add kured

# This is the commit message Azure#21:

change

# This is the commit message Azure#22:

changes

# This is the commit message Azure#23:

add default namespace

# This is the commit message Azure#24:

change

# This is the commit message Azure#25:

add integration test

# This is the commit message Azure#26:

changes

# This is the commit message Azure#27:

add helm test

# This is the commit message Azure#28:

change print statement to error

# This is the commit message Azure#29:

change

# This is the commit message Azure#30:

more changes

# This is the commit message Azure#31:

add go installation

# This is the commit message Azure#32:

fix unit test

# This is the commit message Azure#33:

iptables to Helm

# This is the commit message Azure#34:

add custom resource collector

# This is the commit message Azure#35:

add new exporter, diagnoser, collector

# This is the commit message Azure#36:

comment unused variables

# This is the commit message Azure#37:

debug exporter

# This is the commit message Azure#38:

filenames

# This is the commit message Azure#39:

test zip function

# This is the commit message Azure#40:

list files

# This is the commit message Azure#41:

fmt to log

# This is the commit message Azure#42:

delete lines

# This is the commit message Azure#43:

changed

# This is the commit message Azure#44:

get current directory

# This is the commit message Azure#45:

remove some print statements

# This is the commit message Azure#46:

test zip

# This is the commit message Azure#47:

changes

# This is the commit message Azure#48:

add windir check

# This is the commit message Azure#49:

minor fix

# This is the commit message Azure#50:

get hostname

# This is the commit message Azure#51:

add expose in dockerfile

# This is the commit message Azure#52:

add exec collector

# This is the commit message Azure#53:

mitigate exit code 126

# This is the commit message Azure#54:

change curl url from example.com to dp endpoint

# This is the commit message Azure#55:

changes

# This is the commit message Azure#56:

uncomment exec

# This is the commit message Azure#57:

add new diagnoser

# This is the commit message Azure#58:

debugging

# This is the commit message Azure#59:

debug

# This is the commit message Azure#60:

debugging

# This is the commit message Azure#61:

remove print statements

# This is the commit message Azure#62:

remove print

# This is the commit message Azure#63:

add back crd print statement

# This is the commit message Azure#64:

change

# This is the commit message Azure#65:

change

# This is the commit message Azure#66:

update dataPoint name

# This is the commit message Azure#67:

modify forloop

# This is the commit message Azure#68:

add filename to datapoint

# This is the commit message Azure#69:

add back log prints

# This is the commit message Azure#70:

test

# This is the commit message Azure#71:

add fields to diagnostic signal

# This is the commit message Azure#72:

add config content to diagnoser

# This is the commit message Azure#73:

change format from yaml to json

# This is the commit message Azure#74:

add parameters for kubeobject config map

# This is the commit message Azure#75:

Revert "introduce arcmode"

This reverts commit 5f4fed4.

# This is the commit message Azure#76:

fix helm collector style

# This is the commit message Azure#77:

revert changes that test arc customizations

# This is the commit message Azure#78:

fix merge conflicts

# This is the commit message Azure#79:

fix merge conflicts

# This is the commit message Azure#80:

Revert "Add v0.3 acr image for Private cluster fix. (Azure#22)"

This reverts commit 49dd302.

# This is the commit message Azure#81:

fix merge conflicts

# This is the commit message Azure#82:

fix merge conflicts

# This is the commit message Azure#83:

add print statement

# This is the commit message Azure#84:

update print statement

# This is the commit message Azure#85:

committed

# This is the commit message Azure#86:

committed

# This is the commit message Azure#87:

committed

# This is the commit message Azure#88:

committed

# This is the commit message Azure#89:

remove print statements

# This is the commit message Azure#90:

fix merge conflicts

# This is the commit message Azure#91:

fix merge conflicts

# This is the commit message Azure#92:

fix merge conflicts

# This is the commit message Azure#93:

add repo command

# This is the commit message Azure#94:

change stable repo name

# This is the commit message Azure#95:

add write to file

# This is the commit message Azure#96:

add kured

# This is the commit message Azure#97:

change

# This is the commit message Azure#98:

changes

# This is the commit message Azure#99:

add default namespace

# This is the commit message Azure#100:

change

# This is the commit message Azure#101:

add integration test

# This is the commit message Azure#102:

changes

# This is the commit message Azure#103:

add helm test

# This is the commit message Azure#104:

change print statement to error

# This is the commit message Azure#105:

change

# This is the commit message Azure#106:

more changes

# This is the commit message Azure#107:

add go installation

# This is the commit message Azure#108:

fix unit test

# This is the commit message Azure#109:

add custom resource collector

# This is the commit message Azure#110:

fix merge conflicts

# This is the commit message Azure#111:

comment unused variables

# This is the commit message Azure#112:

debug exporter

# This is the commit message Azure#113:

filenames

# This is the commit message Azure#114:

test zip function

# This is the commit message Azure#115:

list files

# This is the commit message Azure#116:

fmt to log

# This is the commit message Azure#117:

delete lines

# This is the commit message Azure#118:

changed

# This is the commit message Azure#119:

get current directory

# This is the commit message Azure#120:

remove some print statements

# This is the commit message Azure#121:

test zip

# This is the commit message Azure#122:

changes

# This is the commit message Azure#123:

add windir check

# This is the commit message Azure#124:

minor fix

# This is the commit message Azure#125:

get hostname

# This is the commit message Azure#126:

add expose in dockerfile

# This is the commit message Azure#127:

add exec collector

Author: Sophie Zhao

builder/Dockerfile

@@ -6,9 +6,15 @@ RUN CGO_ENABLED=0 GOOS=linux go build -mod=vendor github.com/Azure/aks-periscope
 
 FROM alpine
 RUN apk --no-cache add ca-certificates
+RUN apk add curl openssl bash --no-cache
 ADD https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl
 RUN chmod +x /usr/local/bin/kubectl
+RUN curl -fsSl -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 
+RUN chmod +x get_helm.sh 
+RUN ./get_helm.sh
+
 RUN mkdir /app
 WORKDIR /app
 COPY --from=builder /app/aks-periscope .
+EXPOSE 8080
 CMD ["./aks-periscope"]

cmd/aks-periscope/aks-periscope.go

@@ -15,6 +15,7 @@ import (
 
 func main() {
 	zipAndExportMode := true
+	//exporter := &exporter.LocalMachineExporter{}
 	exporter := &exporter.AzureBlobExporter{}
 	var waitgroup sync.WaitGroup
 
@@ -23,8 +24,10 @@ func main() {
 		log.Printf("Failed to create CRD: %+v", err)
 	}
 
-	clusterType := strings.ToLower(os.Getenv("CLUSTER_TYPE"))
+	clusterType := os.Getenv("CLUSTER_TYPE")
 
+	storageAccount := os.Getenv("AZURE_BLOB_ACCOUNT_NAME")
+	log.Printf("Storage Account: %s", storageAccount)
 	collectors := []interfaces.Collector{}
 	containerLogsCollector := collector.NewContainerLogsCollector(exporter)
 	collectors = append(collectors, containerLogsCollector)
@@ -41,7 +44,15 @@ func main() {
 	kubeletCmdCollector := collector.NewKubeletCmdCollector(exporter)
 	systemPerfCollector := collector.NewSystemPerfCollector(exporter)
 
-	if clusterType != "connectedcluster" {
+	execCollector := collector.NewExecCollector(exporter)
+	helmCollector := collector.NewHelmCollector(exporter)
+	customResourceCollector := collector.NewCustomResourceCollector(exporter)
+
+	if clusterType == "connectedcluster" {
+		collectors = append(collectors, helmCollector)
+		collectors = append(collectors, execCollector)
+		collectors = append(collectors, customResourceCollector)
+	} else {
 		collectors = append(collectors, systemLogsCollector)
 		collectors = append(collectors, ipTablesCollector)
 		collectors = append(collectors, nodeLogsCollector)
@@ -63,6 +74,7 @@ func main() {
 			if err != nil {
 				log.Printf("Collector: %s, export data failed: %+v\n", c.GetName(), err)
 			}
+
 			waitgroup.Done()
 		}(c)
 	}

deployment/aks-periscope.yaml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: azure-k8-periscope
+  name: aks-periscope
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: aks-periscope-service-account
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -28,7 +28,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: aks-periscope-service-account
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 roleRef:
   kind: ClusterRole
   name: aks-periscope-role
@@ -41,7 +41,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: aks-periscope-service-account
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 roleRef:
   kind: ClusterRole
   name: view
@@ -50,18 +50,18 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: azure-k8-periscope
-  namespace: azure-k8-periscope
+  name: aks-periscope
+  namespace: aks-periscope
   labels:
-    app: azure-k8-periscope
+    app: aks-periscope
 spec:
   selector:
     matchLabels:
-      app: azure-k8-periscope
+      app: aks-periscope
   template:
     metadata:
       labels:
-        app: azure-k8-periscope
+        app: aks-periscope
     spec:
       serviceAccountName: aks-periscope-service-account
       hostPID: true
@@ -80,6 +80,8 @@ spec:
             name: kubeobjects-config
         - configMapRef:
             name: nodelogs-config
+        - configMapRef:
+            name: clustertype-config
         - secretRef:
             name: azureblob-secret
         volumeMounts:
@@ -102,43 +104,43 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: azureblob-secret
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 type: Opaque
 data:
-  AZURE_BLOB_ACCOUNT_NAME: Y2Nzb3BoaWV6aGFvYXJuc2FlYXN0dXMy
-  AZURE_BLOB_SAS_KEY: P3N2PTIwMjAtMDItMTAmc3M9YmZxdCZzcnQ9c2NvJnNwPXJ3ZGxhY3VweCZzZT0yMDIxLTA0LTIwVDAwOjA5OjM0WiZzdD0yMDIxLTA0LTE5VDE2OjA5OjM0WiZzcHI9aHR0cHMmc2lnPXZrM05FZ2ZBQm93a0RoQXJ5Rkd6b0NHJTJGRDB5SGUlMkJNYkg1SG82SmR1dmFrJTNE
+  AZURE_BLOB_ACCOUNT_NAME: # <accountName, base64 encoded>
+  AZURE_BLOB_SAS_KEY: # <saskey, base64 encoded>
 ---
 apiVersion: v1 
 kind: ConfigMap 
 metadata:
   name: containerlogs-config
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 data:
   DIAGNOSTIC_CONTAINERLOGS_LIST: azure-arc
 ---
 apiVersion: v1 
 kind: ConfigMap 
 metadata:
   name: kubeobjects-config
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 data:
   DIAGNOSTIC_KUBEOBJECTS_LIST: azure-arc/pod azure-arc/service azure-arc/deployment
 ---
 apiVersion: v1 
 kind: ConfigMap 
 metadata:
   name: nodelogs-config
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 data:
   DIAGNOSTIC_NODELOGS_LIST: /var/log/azure/cluster-provision.log /var/log/cloud-init.log
 ---
 apiVersion: v1 
 kind: ConfigMap 
 metadata:
   name: clustertype-config
-  namespace: azure-k8-periscope
+  namespace: aks-periscope
 data:
-  CLUSTER_TYPE: # <cluster type, options: ConnectedCluster, ManagedCluster, ArcAppliances, etc.>
+  CLUSTER_TYPE: connectedcluster
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
@@ -167,4 +169,4 @@ spec:
     singular: diagnostic
     kind: Diagnostic
     shortNames:
-    - apd
+    - apd

integration/Dockerfile

@@ -0,0 +1,25 @@
+FROM golang AS builder
+RUN mkdir /app
+ADD . /app
+WORKDIR /app
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=vendor github.com/Azure/aks-periscope/cmd/aks-periscope
+
+FROM alpine
+RUN apk --no-cache add ca-certificates
+RUN apk add curl openssl bash --no-cache
+ADD https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl
+RUN chmod +x /usr/local/bin/kubectl
+RUN curl -fsSl -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 
+RUN chmod +x get_helm.sh 
+RUN ./get_helm.sh
+
+RUN apk add --no-cache git make musl-dev go
+
+# Configure Go
+ENV GOROOT /usr/lib/go
+ENV GOPATH /go
+ENV PATH /go/bin:$PATH
+
+RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
+WORKDIR /tests
+CMD CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./...

pkg/collector/collector.go

@@ -12,6 +12,8 @@ const (
 	ContainerLogs
 	//CustomResource defines CustomResource Collector Type
 	CustomResource
+	//CustomResource defines CustomResource Collector Type
+	CustomResourceV2
 	//Exec defines Exec Collector Type
 	Exec
 	//Helm defines Helm Collector Type
@@ -34,7 +36,7 @@ const (
 
 // Name returns type name
 func (t Type) name() string {
-	return [...]string{"dns", "containerlogs", "customresource", "exec", "helm", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t]
+	return [...]string{"dns", "containerlogs", "customresource", "customresourcev2", "exec", "helm", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t]
 }
 
 // BaseCollector defines Base Collector

pkg/collector/customresource_collector.go

@@ -1,7 +1,11 @@
 package collector
 
 import (
+	"path/filepath"
+	"strings"
+
 	"github.com/Azure/aks-periscope/pkg/interfaces"
+	"github.com/Azure/aks-periscope/pkg/utils"
 )
 
 // CustomResourceCollector defines a CustomResources Collector struct
@@ -23,5 +27,33 @@ func NewCustomResourceCollector(exporter interfaces.Exporter) *CustomResourceCol
 
 // Collect implements the interface method
 func (collector *CustomResourceCollector) Collect() error {
+	rootPath, err := utils.CreateCollectorDir(collector.GetName())
+
+	output, err := utils.RunCommandOnContainer("kubectl", "get", "namespace", "--output=jsonpath={.items..metadata.name}")
+	if err != nil {
+		return err
+	}
+	namespaces := strings.Split(output, " ")
+	for _, namespace := range namespaces {
+		output, err = utils.RunCommandOnContainer("kubectl", "-n", namespace, "get", "crd", "--output=jsonpath={.items..metadata.name}")
+		if err != nil {
+			return err
+		}
+
+		objects := strings.Split(output, " ")
+		for _, object := range objects {
+			customResourceFile := filepath.Join(rootPath, namespace+"_"+"crd"+"_"+object)
+			output, err := utils.RunCommandOnContainer("kubectl", "-n", namespace, "describe", "crd", object)
+			if err != nil {
+				return err
+			}
+			err = utils.WriteToFile(customResourceFile, output)
+			if err != nil {
+				return err
+			}
+
+			collector.AddToCollectorFiles(customResourceFile)
+		}
+	}
 	return nil
 }

pkg/collector/exec_collector.go

@@ -1,7 +1,12 @@
 package collector
 
 import (
+	"os"
+	"path/filepath"
+	"strings"
+
 	"github.com/Azure/aks-periscope/pkg/interfaces"
+	"github.com/Azure/aks-periscope/pkg/utils"
 )
 
 // ExecCollector defines a Exec Collector struct
@@ -23,5 +28,31 @@ func NewExecCollector(exporter interfaces.Exporter) *ExecCollector {
 
 // Collect implements the interface method
 func (collector *ExecCollector) Collect() error {
+	rootPath, err := utils.CreateCollectorDir(collector.GetName())
+	if err != nil {
+		return err
+	}
+	namespaces := strings.Fields(os.Getenv("DIAGNOSTIC_EXEC_LIST"))
+	for _, namespace := range namespaces {
+		output, err := utils.RunCommandOnContainer("kubectl", "get", "-n", namespace, "pods", "--output=jsonpath={.items..metadata.name}")
+		if err != nil {
+			return err
+		}
+		pods := strings.Split(output, " ")
+
+		for _, pod := range pods {
+			execLog := filepath.Join(rootPath, namespace+"_"+pod)
+			output, err := utils.RunCommandOnContainer("kubectl", "-n", namespace, "exec", pod, "--", "curl", "example.com")
+			if err != nil {
+				return err
+			}
+			err = utils.WriteToFile(execLog, output)
+			if err != nil {
+				return err
+			}
+
+			collector.AddToCollectorFiles(execLog)
+		}
+	}
 	return nil
 }

pkg/collector/helm_collector.go

@@ -1,15 +1,18 @@
 package collector
 
 import (
+	"path/filepath"
+
 	"github.com/Azure/aks-periscope/pkg/interfaces"
+	"github.com/Azure/aks-periscope/pkg/utils"
 )
 
 // HelmCollector defines a Helm Collector struct
 type HelmCollector struct {
 	BaseCollector
 }
 
-var _ interfaces.Collector = &IPTablesCollector{}
+var _ interfaces.Collector = &HelmCollector{}
 
 // NewHelmCollector is a constructor
 func NewHelmCollector(exporter interfaces.Exporter) *HelmCollector {
@@ -23,5 +26,43 @@ func NewHelmCollector(exporter interfaces.Exporter) *HelmCollector {
 
 // Collect implements the interface method
 func (collector *HelmCollector) Collect() error {
+	rootPath, err := utils.CreateCollectorDir(collector.GetName())
+	if err != nil {
+		return err
+	}
+	helmListFile := filepath.Join(rootPath, "helm_list")
+	output, err := utils.RunCommandOnContainer("helm", "list", "--all-namespaces")
+	if err != nil {
+		return err
+	}
+
+	err = utils.WriteToFile(helmListFile, output)
+	if err != nil {
+		return err
+	}
+
+	collector.AddToCollectorFiles(helmListFile)
+
+	helmHistoryFile := filepath.Join(rootPath, collector.GetName())
+	output, err = utils.RunCommandOnContainer("helm", "history", "-n", "default", "azure-arc")
+	if err != nil {
+		return err
+	}
+	err = utils.WriteToFile(helmHistoryFile, output)
+	if err != nil {
+		return err
+	}
+
+	helmHistoryFile := filepath.Join(rootPath, collector.GetName())
+	output, err = utils.RunCommandOnContainer("helm", "history", "-n", "default", "azure-arc")
+	if err != nil {
+		return err
+	}
+	err = utils.WriteToFile(helmHistoryFile, output)
+	if err != nil {
+		return err
+	}
+
+	collector.AddToCollectorFiles(helmHistoryFile)
 	return nil
 }